2024-11-27 00:14:07,775 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 00:14:07,789 main DEBUG Took 0.011974 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 00:14:07,790 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 00:14:07,790 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 00:14:07,792 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 00:14:07,793 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,800 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 00:14:07,813 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,815 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,816 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,817 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,818 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,819 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,820 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,820 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,821 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,821 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,822 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,823 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,824 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 00:14:07,824 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,825 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 00:14:07,826 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 00:14:07,828 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 00:14:07,830 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 00:14:07,830 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 00:14:07,832 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 00:14:07,832 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 00:14:07,841 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 00:14:07,843 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 00:14:07,845 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 00:14:07,845 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 00:14:07,846 main DEBUG createAppenders(={Console}) 2024-11-27 00:14:07,847 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-27 00:14:07,847 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 00:14:07,847 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-27 00:14:07,848 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 00:14:07,848 main DEBUG OutputStream closed 2024-11-27 00:14:07,849 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 00:14:07,849 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 00:14:07,849 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-27 00:14:07,925 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 00:14:07,927 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 00:14:07,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 00:14:07,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 00:14:07,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 00:14:07,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 00:14:07,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 00:14:07,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 00:14:07,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 00:14:07,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 00:14:07,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 00:14:07,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 00:14:07,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 00:14:07,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 00:14:07,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 00:14:07,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 00:14:07,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 00:14:07,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 00:14:07,938 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 00:14:07,939 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-27 00:14:07,939 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 00:14:07,940 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-27T00:14:08,168 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1 2024-11-27 00:14:08,171 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 00:14:08,171 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T00:14:08,182 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-27T00:14:08,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T00:14:08,224 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3, deleteOnExit=true 2024-11-27T00:14:08,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T00:14:08,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/test.cache.data in system properties and HBase conf 2024-11-27T00:14:08,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T00:14:08,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/hadoop.log.dir in system properties and HBase conf 2024-11-27T00:14:08,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T00:14:08,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T00:14:08,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T00:14:08,340 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T00:14:08,454 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T00:14:08,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T00:14:08,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T00:14:08,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T00:14:08,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T00:14:08,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T00:14:08,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T00:14:08,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T00:14:08,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T00:14:08,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T00:14:08,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/nfs.dump.dir in system properties and HBase conf 2024-11-27T00:14:08,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/java.io.tmpdir in system properties and HBase conf 2024-11-27T00:14:08,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T00:14:08,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T00:14:08,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T00:14:09,787 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T00:14:09,855 INFO [Time-limited test {}] log.Log(170): Logging initialized @2735ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T00:14:09,921 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T00:14:09,980 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T00:14:10,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T00:14:10,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T00:14:10,002 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T00:14:10,014 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T00:14:10,017 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/hadoop.log.dir/,AVAILABLE} 2024-11-27T00:14:10,018 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T00:14:10,193 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/java.io.tmpdir/jetty-localhost-43535-hadoop-hdfs-3_4_1-tests_jar-_-any-6156256150206287776/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T00:14:10,200 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43535} 2024-11-27T00:14:10,200 INFO [Time-limited test {}] server.Server(415): Started @3081ms 2024-11-27T00:14:10,832 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T00:14:10,841 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T00:14:10,842 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T00:14:10,843 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T00:14:10,843 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T00:14:10,844 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/hadoop.log.dir/,AVAILABLE} 2024-11-27T00:14:10,845 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T00:14:10,959 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/java.io.tmpdir/jetty-localhost-34905-hadoop-hdfs-3_4_1-tests_jar-_-any-6759742998261839507/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T00:14:10,960 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34905} 2024-11-27T00:14:10,960 INFO [Time-limited test {}] server.Server(415): Started @3841ms 2024-11-27T00:14:11,011 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T00:14:12,040 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/data/data2/current/BP-1139848992-172.17.0.3-1732666449046/current, will proceed with Du for space computation calculation, 2024-11-27T00:14:12,040 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/data/data1/current/BP-1139848992-172.17.0.3-1732666449046/current, will proceed with Du for space computation calculation, 2024-11-27T00:14:12,064 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T00:14:12,105 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5980321fc0e3126d with lease ID 0xf9061afc12968442: Processing first storage report for DS-fe5d143d-76ca-4a9d-9255-a18c0e781b7d from datanode DatanodeRegistration(127.0.0.1:37783, datanodeUuid=955c775f-b623-4cac-9f03-56e55d61fb75, infoPort=40321, infoSecurePort=0, ipcPort=44559, storageInfo=lv=-57;cid=testClusterID;nsid=736021246;c=1732666449046) 2024-11-27T00:14:12,106 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5980321fc0e3126d with lease ID 0xf9061afc12968442: from storage DS-fe5d143d-76ca-4a9d-9255-a18c0e781b7d node DatanodeRegistration(127.0.0.1:37783, datanodeUuid=955c775f-b623-4cac-9f03-56e55d61fb75, infoPort=40321, infoSecurePort=0, ipcPort=44559, storageInfo=lv=-57;cid=testClusterID;nsid=736021246;c=1732666449046), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T00:14:12,107 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5980321fc0e3126d with lease ID 0xf9061afc12968442: Processing first storage report for DS-864c43c5-0c27-4d3a-b881-0dc5113cc05b from datanode DatanodeRegistration(127.0.0.1:37783, datanodeUuid=955c775f-b623-4cac-9f03-56e55d61fb75, infoPort=40321, infoSecurePort=0, ipcPort=44559, storageInfo=lv=-57;cid=testClusterID;nsid=736021246;c=1732666449046) 2024-11-27T00:14:12,107 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5980321fc0e3126d with lease ID 0xf9061afc12968442: from storage DS-864c43c5-0c27-4d3a-b881-0dc5113cc05b node DatanodeRegistration(127.0.0.1:37783, datanodeUuid=955c775f-b623-4cac-9f03-56e55d61fb75, infoPort=40321, infoSecurePort=0, ipcPort=44559, storageInfo=lv=-57;cid=testClusterID;nsid=736021246;c=1732666449046), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T00:14:12,123 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1 2024-11-27T00:14:12,217 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/zookeeper_0, clientPort=56431, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T00:14:12,230 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56431 2024-11-27T00:14:12,245 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T00:14:12,247 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T00:14:12,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741825_1001 (size=7) 2024-11-27T00:14:12,845 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 with version=8 2024-11-27T00:14:12,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/hbase-staging 2024-11-27T00:14:12,931 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T00:14:13,268 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f036bc4986ea:0 server-side Connection retries=45 2024-11-27T00:14:13,277 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T00:14:13,277 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T00:14:13,282 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T00:14:13,282 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T00:14:13,283 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T00:14:13,409 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T00:14:13,458 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T00:14:13,466 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T00:14:13,470 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T00:14:13,492 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4685 (auto-detected) 2024-11-27T00:14:13,493 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-27T00:14:13,509 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33599 2024-11-27T00:14:13,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33599 connecting to ZooKeeper ensemble=127.0.0.1:56431 2024-11-27T00:14:13,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335990x0, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T00:14:13,677 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33599-0x10179a3e48a0000 connected 2024-11-27T00:14:13,777 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T00:14:13,781 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T00:14:13,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T00:14:13,794 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772, hbase.cluster.distributed=false 2024-11-27T00:14:13,815 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T00:14:13,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33599 2024-11-27T00:14:13,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33599 2024-11-27T00:14:13,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33599 2024-11-27T00:14:13,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33599 2024-11-27T00:14:13,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33599 2024-11-27T00:14:13,923 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f036bc4986ea:0 server-side Connection retries=45 2024-11-27T00:14:13,924 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T00:14:13,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T00:14:13,925 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T00:14:13,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T00:14:13,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T00:14:13,928 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T00:14:13,931 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T00:14:13,932 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33881 2024-11-27T00:14:13,934 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33881 connecting to ZooKeeper ensemble=127.0.0.1:56431 2024-11-27T00:14:13,935 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T00:14:13,939 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T00:14:13,954 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338810x0, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T00:14:13,955 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T00:14:13,955 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33881-0x10179a3e48a0001 connected 2024-11-27T00:14:13,958 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T00:14:13,966 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T00:14:13,969 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T00:14:13,974 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T00:14:13,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33881 2024-11-27T00:14:13,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33881 2024-11-27T00:14:13,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33881 2024-11-27T00:14:13,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33881 2024-11-27T00:14:13,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33881 2024-11-27T00:14:13,994 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f036bc4986ea:33599 2024-11-27T00:14:13,995 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f036bc4986ea,33599,1732666453115 2024-11-27T00:14:14,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T00:14:14,011 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T00:14:14,013 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f036bc4986ea,33599,1732666453115 2024-11-27T00:14:14,042 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T00:14:14,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:14,043 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:14,044 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T00:14:14,045 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f036bc4986ea,33599,1732666453115 from backup master directory 2024-11-27T00:14:14,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f036bc4986ea,33599,1732666453115 2024-11-27T00:14:14,059 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T00:14:14,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T00:14:14,059 WARN [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T00:14:14,060 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f036bc4986ea,33599,1732666453115 2024-11-27T00:14:14,062 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T00:14:14,064 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T00:14:14,114 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/hbase.id] with ID: 8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0 2024-11-27T00:14:14,114 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/.tmp/hbase.id 2024-11-27T00:14:14,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741826_1002 (size=42) 2024-11-27T00:14:14,529 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/.tmp/hbase.id]:[hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/hbase.id] 2024-11-27T00:14:14,577 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T00:14:14,582 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T00:14:14,600 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-27T00:14:14,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:14,650 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:14,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741827_1003 (size=196) 2024-11-27T00:14:15,085 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:14:15,088 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T00:14:15,101 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:15,104 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T00:14:15,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741828_1004 (size=1189) 2024-11-27T00:14:15,549 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store 2024-11-27T00:14:15,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741829_1005 (size=34) 2024-11-27T00:14:15,972 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T00:14:15,976 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:15,978 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T00:14:15,978 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T00:14:15,979 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T00:14:15,981 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T00:14:15,981 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T00:14:15,981 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T00:14:15,983 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732666455978Disabling compacts and flushes for region at 1732666455978Disabling writes for close at 1732666455981 (+3 ms)Writing region close event to WAL at 1732666455981Closed at 1732666455981 2024-11-27T00:14:15,985 WARN [master/f036bc4986ea:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/.initializing 2024-11-27T00:14:15,985 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/WALs/f036bc4986ea,33599,1732666453115 2024-11-27T00:14:15,993 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T00:14:16,006 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f036bc4986ea%2C33599%2C1732666453115, suffix=, logDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/WALs/f036bc4986ea,33599,1732666453115, archiveDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/oldWALs, maxLogs=10 2024-11-27T00:14:16,026 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/WALs/f036bc4986ea,33599,1732666453115/f036bc4986ea%2C33599%2C1732666453115.1732666456010, exclude list is [], retry=0 2024-11-27T00:14:16,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37783,DS-fe5d143d-76ca-4a9d-9255-a18c0e781b7d,DISK] 2024-11-27T00:14:16,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T00:14:16,080 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/WALs/f036bc4986ea,33599,1732666453115/f036bc4986ea%2C33599%2C1732666453115.1732666456010 2024-11-27T00:14:16,081 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40321:40321)] 2024-11-27T00:14:16,082 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:14:16,082 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:16,085 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,086 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T00:14:16,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:16,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:16,151 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T00:14:16,154 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:16,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:16,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T00:14:16,158 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:16,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:16,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T00:14:16,162 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:16,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:16,164 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,167 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,169 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,174 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,175 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,179 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T00:14:16,183 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T00:14:16,187 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T00:14:16,189 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68146563, jitterRate=0.015462920069694519}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T00:14:16,195 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732666456098Initializing all the Stores at 1732666456100 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732666456101 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666456102 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666456102Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666456102Cleaning up temporary data from old regions at 1732666456175 (+73 ms)Region opened successfully at 1732666456195 (+20 ms) 2024-11-27T00:14:16,196 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T00:14:16,228 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140cdd2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f036bc4986ea/172.17.0.3:0 2024-11-27T00:14:16,256 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T00:14:16,265 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T00:14:16,265 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T00:14:16,267 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T00:14:16,268 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T00:14:16,272 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-27T00:14:16,272 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T00:14:16,295 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T00:14:16,303 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T00:14:16,380 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T00:14:16,385 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T00:14:16,388 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T00:14:16,400 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T00:14:16,402 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T00:14:16,405 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T00:14:16,416 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T00:14:16,418 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T00:14:16,427 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T00:14:16,450 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T00:14:16,463 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T00:14:16,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T00:14:16,474 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T00:14:16,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:16,474 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:16,479 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f036bc4986ea,33599,1732666453115, sessionid=0x10179a3e48a0000, setting cluster-up flag (Was=false) 2024-11-27T00:14:16,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:16,511 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:16,548 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T00:14:16,550 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f036bc4986ea,33599,1732666453115 2024-11-27T00:14:16,575 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:16,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:16,610 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T00:14:16,612 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f036bc4986ea,33599,1732666453115 2024-11-27T00:14:16,618 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T00:14:16,684 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(746): ClusterId : 8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0 2024-11-27T00:14:16,686 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T00:14:16,687 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T00:14:16,695 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T00:14:16,697 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T00:14:16,697 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T00:14:16,701 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T00:14:16,707 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T00:14:16,707 DEBUG [RS:0;f036bc4986ea:33881 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a157f06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f036bc4986ea/172.17.0.3:0 2024-11-27T00:14:16,706 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f036bc4986ea,33599,1732666453115 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T00:14:16,713 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f036bc4986ea:0, corePoolSize=5, maxPoolSize=5 2024-11-27T00:14:16,713 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f036bc4986ea:0, corePoolSize=5, maxPoolSize=5 2024-11-27T00:14:16,713 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f036bc4986ea:0, corePoolSize=5, maxPoolSize=5 2024-11-27T00:14:16,713 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f036bc4986ea:0, corePoolSize=5, maxPoolSize=5 2024-11-27T00:14:16,713 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f036bc4986ea:0, corePoolSize=10, maxPoolSize=10 2024-11-27T00:14:16,713 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,714 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f036bc4986ea:0, corePoolSize=2, maxPoolSize=2 2024-11-27T00:14:16,714 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,717 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732666486717 2024-11-27T00:14:16,719 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T00:14:16,720 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T00:14:16,720 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T00:14:16,721 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T00:14:16,721 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f036bc4986ea:33881 2024-11-27T00:14:16,724 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T00:14:16,725 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T00:14:16,725 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T00:14:16,725 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T00:14:16,725 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T00:14:16,726 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T00:14:16,726 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T00:14:16,727 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:16,727 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(2659): reportForDuty to master=f036bc4986ea,33599,1732666453115 with port=33881, startcode=1732666453890 2024-11-27T00:14:16,727 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T00:14:16,726 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,730 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T00:14:16,731 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T00:14:16,731 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T00:14:16,734 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T00:14:16,734 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T00:14:16,736 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f036bc4986ea:0:becomeActiveMaster-HFileCleaner.large.0-1732666456735,5,FailOnTimeoutGroup] 2024-11-27T00:14:16,737 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f036bc4986ea:0:becomeActiveMaster-HFileCleaner.small.0-1732666456737,5,FailOnTimeoutGroup] 2024-11-27T00:14:16,737 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,737 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T00:14:16,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741831_1007 (size=1321) 2024-11-27T00:14:16,739 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,739 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,740 DEBUG [RS:0;f036bc4986ea:33881 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T00:14:16,766 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T00:14:16,771 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33599 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f036bc4986ea,33881,1732666453890 2024-11-27T00:14:16,772 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33599 {}] master.ServerManager(517): Registering regionserver=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:16,784 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 2024-11-27T00:14:16,784 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40817 2024-11-27T00:14:16,784 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T00:14:16,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T00:14:16,796 DEBUG [RS:0;f036bc4986ea:33881 {}] zookeeper.ZKUtil(111): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f036bc4986ea,33881,1732666453890 2024-11-27T00:14:16,796 WARN [RS:0;f036bc4986ea:33881 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T00:14:16,797 INFO [RS:0;f036bc4986ea:33881 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T00:14:16,797 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/WALs/f036bc4986ea,33881,1732666453890 2024-11-27T00:14:16,799 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f036bc4986ea,33881,1732666453890] 2024-11-27T00:14:16,825 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T00:14:16,840 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T00:14:16,845 INFO [RS:0;f036bc4986ea:33881 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T00:14:16,845 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,846 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T00:14:16,852 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T00:14:16,853 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,854 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,854 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,854 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,854 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,854 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,854 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f036bc4986ea:0, corePoolSize=2, maxPoolSize=2 2024-11-27T00:14:16,855 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,855 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,855 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,855 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,855 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,855 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f036bc4986ea:0, corePoolSize=1, maxPoolSize=1 2024-11-27T00:14:16,855 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f036bc4986ea:0, corePoolSize=3, maxPoolSize=3 2024-11-27T00:14:16,856 DEBUG [RS:0;f036bc4986ea:33881 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0, corePoolSize=3, maxPoolSize=3 2024-11-27T00:14:16,856 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,857 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,857 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,857 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,857 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,857 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=f036bc4986ea,33881,1732666453890-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T00:14:16,874 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T00:14:16,876 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=f036bc4986ea,33881,1732666453890-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,876 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,876 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.Replication(171): f036bc4986ea,33881,1732666453890 started 2024-11-27T00:14:16,893 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:16,893 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1482): Serving as f036bc4986ea,33881,1732666453890, RpcServer on f036bc4986ea/172.17.0.3:33881, sessionid=0x10179a3e48a0001 2024-11-27T00:14:16,894 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T00:14:16,894 DEBUG [RS:0;f036bc4986ea:33881 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f036bc4986ea,33881,1732666453890 2024-11-27T00:14:16,894 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f036bc4986ea,33881,1732666453890' 2024-11-27T00:14:16,894 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T00:14:16,895 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T00:14:16,896 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T00:14:16,896 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T00:14:16,896 DEBUG [RS:0;f036bc4986ea:33881 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f036bc4986ea,33881,1732666453890 2024-11-27T00:14:16,896 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f036bc4986ea,33881,1732666453890' 2024-11-27T00:14:16,896 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T00:14:16,897 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T00:14:16,898 DEBUG [RS:0;f036bc4986ea:33881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T00:14:16,898 INFO [RS:0;f036bc4986ea:33881 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T00:14:16,898 INFO [RS:0;f036bc4986ea:33881 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T00:14:17,008 INFO [RS:0;f036bc4986ea:33881 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T00:14:17,013 INFO [RS:0;f036bc4986ea:33881 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f036bc4986ea%2C33881%2C1732666453890, suffix=, logDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/WALs/f036bc4986ea,33881,1732666453890, archiveDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/oldWALs, maxLogs=32 2024-11-27T00:14:17,029 DEBUG [RS:0;f036bc4986ea:33881 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/WALs/f036bc4986ea,33881,1732666453890/f036bc4986ea%2C33881%2C1732666453890.1732666457016, exclude list is [], retry=0 2024-11-27T00:14:17,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37783,DS-fe5d143d-76ca-4a9d-9255-a18c0e781b7d,DISK] 2024-11-27T00:14:17,037 INFO [RS:0;f036bc4986ea:33881 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/WALs/f036bc4986ea,33881,1732666453890/f036bc4986ea%2C33881%2C1732666453890.1732666457016 2024-11-27T00:14:17,038 DEBUG [RS:0;f036bc4986ea:33881 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40321:40321)] 2024-11-27T00:14:17,143 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T00:14:17,144 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 2024-11-27T00:14:17,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741833_1009 (size=32) 2024-11-27T00:14:17,563 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:17,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T00:14:17,571 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T00:14:17,571 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:17,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:17,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T00:14:17,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T00:14:17,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:17,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:17,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T00:14:17,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T00:14:17,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:17,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:17,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T00:14:17,584 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T00:14:17,584 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:17,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:17,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T00:14:17,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740 2024-11-27T00:14:17,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740 2024-11-27T00:14:17,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T00:14:17,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T00:14:17,591 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:14:17,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T00:14:17,598 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T00:14:17,599 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62873973, jitterRate=-0.0631047934293747}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:14:17,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732666457563Initializing all the Stores at 1732666457567 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732666457567Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732666457567Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666457567Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732666457567Cleaning up temporary data from old regions at 1732666457591 (+24 ms)Region opened successfully at 1732666457602 (+11 ms) 2024-11-27T00:14:17,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T00:14:17,602 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T00:14:17,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T00:14:17,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T00:14:17,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T00:14:17,604 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T00:14:17,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732666457602Disabling compacts and flushes for region at 1732666457602Disabling writes for close at 1732666457603 (+1 ms)Writing region close event to WAL at 1732666457604 (+1 ms)Closed at 1732666457604 2024-11-27T00:14:17,608 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T00:14:17,608 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T00:14:17,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T00:14:17,621 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T00:14:17,624 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T00:14:17,782 DEBUG [f036bc4986ea:33599 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T00:14:17,796 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:17,801 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f036bc4986ea,33881,1732666453890, state=OPENING 2024-11-27T00:14:17,853 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T00:14:17,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:17,863 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:14:17,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T00:14:17,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T00:14:17,868 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T00:14:17,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:14:18,057 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T00:14:18,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40781, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T00:14:18,072 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T00:14:18,072 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T00:14:18,073 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T00:14:18,076 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f036bc4986ea%2C33881%2C1732666453890.meta, suffix=.meta, logDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/WALs/f036bc4986ea,33881,1732666453890, archiveDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/oldWALs, maxLogs=32 2024-11-27T00:14:18,091 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/WALs/f036bc4986ea,33881,1732666453890/f036bc4986ea%2C33881%2C1732666453890.meta.1732666458079.meta, exclude list is [], retry=0 2024-11-27T00:14:18,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37783,DS-fe5d143d-76ca-4a9d-9255-a18c0e781b7d,DISK] 2024-11-27T00:14:18,099 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/WALs/f036bc4986ea,33881,1732666453890/f036bc4986ea%2C33881%2C1732666453890.meta.1732666458079.meta 2024-11-27T00:14:18,099 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40321:40321)] 2024-11-27T00:14:18,099 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:14:18,101 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T00:14:18,103 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T00:14:18,107 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T00:14:18,111 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T00:14:18,112 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:18,112 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T00:14:18,112 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T00:14:18,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T00:14:18,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T00:14:18,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:18,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:18,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T00:14:18,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T00:14:18,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:18,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:18,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T00:14:18,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T00:14:18,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:18,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:18,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T00:14:18,124 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T00:14:18,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:18,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T00:14:18,125 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T00:14:18,126 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740 2024-11-27T00:14:18,129 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740 2024-11-27T00:14:18,132 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T00:14:18,132 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T00:14:18,133 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:14:18,136 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T00:14:18,139 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69105739, jitterRate=0.02975575625896454}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:14:18,139 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T00:14:18,142 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732666458113Writing region info on filesystem at 1732666458113Initializing all the Stores at 1732666458115 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732666458115Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732666458115Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666458115Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732666458115Cleaning up temporary data from old regions at 1732666458132 (+17 ms)Running coprocessor post-open hooks at 1732666458139 (+7 ms)Region opened successfully at 1732666458141 (+2 ms) 2024-11-27T00:14:18,148 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732666458047 2024-11-27T00:14:18,158 DEBUG [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T00:14:18,158 INFO [RS_OPEN_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T00:14:18,160 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:18,162 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f036bc4986ea,33881,1732666453890, state=OPEN 2024-11-27T00:14:18,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T00:14:18,262 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T00:14:18,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T00:14:18,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T00:14:18,263 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:18,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T00:14:18,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f036bc4986ea,33881,1732666453890 in 392 msec 2024-11-27T00:14:18,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T00:14:18,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 662 msec 2024-11-27T00:14:18,281 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T00:14:18,281 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T00:14:18,300 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:18,301 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:18,320 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:18,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37649, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:18,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6940 sec 2024-11-27T00:14:18,341 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732666458341, completionTime=-1 2024-11-27T00:14:18,343 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T00:14:18,343 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T00:14:18,367 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T00:14:18,367 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732666518367 2024-11-27T00:14:18,368 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732666578367 2024-11-27T00:14:18,368 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-27T00:14:18,370 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f036bc4986ea,33599,1732666453115-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:18,370 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f036bc4986ea,33599,1732666453115-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:18,370 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f036bc4986ea,33599,1732666453115-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:18,371 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f036bc4986ea:33599, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:18,372 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:18,372 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:18,378 DEBUG [master/f036bc4986ea:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T00:14:18,414 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.354sec 2024-11-27T00:14:18,416 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T00:14:18,417 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T00:14:18,419 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T00:14:18,419 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T00:14:18,419 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T00:14:18,421 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f036bc4986ea,33599,1732666453115-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T00:14:18,421 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f036bc4986ea,33599,1732666453115-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T00:14:18,430 DEBUG [master/f036bc4986ea:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T00:14:18,431 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T00:14:18,432 INFO [master/f036bc4986ea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f036bc4986ea,33599,1732666453115-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T00:14:18,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:18,497 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T00:14:18,497 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T00:14:18,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:18,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:18,508 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed8bbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:18,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:18,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:18,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:18,520 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:18,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:18,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f036bc4986ea,33599,1732666453115 2024-11-27T00:14:18,559 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=3018 2024-11-27T00:14:18,567 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T00:14:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f036bc4986ea,33599,1732666453115 2024-11-27T00:14:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@219792a1 2024-11-27T00:14:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T00:14:18,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T00:14:18,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:14:18,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:14:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:18,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T00:14:18,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-27T00:14:18,604 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:18,607 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T00:14:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T00:14:18,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741835_1011 (size=1061) 2024-11-27T00:14:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T00:14:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T00:14:19,031 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 2024-11-27T00:14:19,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741836_1012 (size=53) 2024-11-27T00:14:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T00:14:19,445 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:19,446 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e7356b094232ddb421ff69892486ac78, disabling compactions & flushes 2024-11-27T00:14:19,446 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:19,446 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:19,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. after waiting 0 ms 2024-11-27T00:14:19,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:19,447 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:19,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7356b094232ddb421ff69892486ac78: Waiting for close lock at 1732666459446Disabling compacts and flushes for region at 1732666459446Disabling writes for close at 1732666459447 (+1 ms)Writing region close event to WAL at 1732666459447Closed at 1732666459447 2024-11-27T00:14:19,452 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T00:14:19,458 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732666459452"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732666459452"}]},"ts":"1732666459452"} 2024-11-27T00:14:19,462 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T00:14:19,464 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T00:14:19,467 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666459464"}]},"ts":"1732666459464"} 2024-11-27T00:14:19,473 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T00:14:19,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, ASSIGN}] 2024-11-27T00:14:19,478 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, ASSIGN 2024-11-27T00:14:19,479 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, ASSIGN; state=OFFLINE, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=false 2024-11-27T00:14:19,631 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7356b094232ddb421ff69892486ac78, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:19,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, ASSIGN because future has completed 2024-11-27T00:14:19,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:14:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T00:14:19,806 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:19,807 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:14:19,807 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,808 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:19,808 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,808 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,812 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,815 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:19,816 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7356b094232ddb421ff69892486ac78 columnFamilyName A 2024-11-27T00:14:19,816 DEBUG [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:19,817 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.HStore(327): Store=e7356b094232ddb421ff69892486ac78/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:19,817 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,819 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:19,820 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7356b094232ddb421ff69892486ac78 columnFamilyName B 2024-11-27T00:14:19,820 DEBUG [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:19,821 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.HStore(327): Store=e7356b094232ddb421ff69892486ac78/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:19,821 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,823 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:19,823 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7356b094232ddb421ff69892486ac78 columnFamilyName C 2024-11-27T00:14:19,824 DEBUG [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:19,824 INFO [StoreOpener-e7356b094232ddb421ff69892486ac78-1 {}] regionserver.HStore(327): Store=e7356b094232ddb421ff69892486ac78/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:19,825 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:19,825 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,827 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,827 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,829 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,829 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,830 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:14:19,833 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,836 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T00:14:19,837 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e7356b094232ddb421ff69892486ac78; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68119946, jitterRate=0.015066295862197876}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:14:19,837 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:19,838 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e7356b094232ddb421ff69892486ac78: Running coprocessor pre-open hook at 1732666459808Writing region info on filesystem at 1732666459808Initializing all the Stores at 1732666459811 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666459811Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666459811Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666459812 (+1 ms)Cleaning up temporary data from old regions at 1732666459829 (+17 ms)Running coprocessor post-open hooks at 1732666459837 (+8 ms)Region opened successfully at 1732666459838 (+1 ms) 2024-11-27T00:14:19,840 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., pid=6, masterSystemTime=1732666459795 2024-11-27T00:14:19,843 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:19,844 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:19,845 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7356b094232ddb421ff69892486ac78, regionState=OPEN, openSeqNum=2, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:19,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:14:19,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T00:14:19,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 in 212 msec 2024-11-27T00:14:19,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T00:14:19,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, ASSIGN in 382 msec 2024-11-27T00:14:19,864 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T00:14:19,865 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666459864"}]},"ts":"1732666459864"} 2024-11-27T00:14:19,868 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T00:14:19,870 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T00:14:19,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2780 sec 2024-11-27T00:14:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T00:14:20,777 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:20,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ecdd502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,795 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175b244f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,796 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,803 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T00:14:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f036bc4986ea,33599,1732666453115 2024-11-27T00:14:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1faa08f6 2024-11-27T00:14:20,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T00:14:20,809 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T00:14:20,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,816 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,822 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a27ead7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,830 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,834 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6d3ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763c2183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,848 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5689a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,851 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b466e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,855 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1b5c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb446c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,861 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276c4225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,863 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f88841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,867 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d0e529e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,869 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306cf84f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:20,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:20,872 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c17951c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:20,875 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315b49bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:20,880 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,880 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,880 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,882 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,887 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,888 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,893 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,896 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,904 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-27T00:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,909 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:20,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,917 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:20,920 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:20,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T00:14:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:14:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:20,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:14:20,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:20,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:20,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:20,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:20,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:20,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T00:14:21,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T00:14:21,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:21,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:21,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/458c746dbf6d459390093caf01561845 is 50, key is test_row_0/A:col10/1732666460969/Put/seqid=0 2024-11-27T00:14:21,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666521093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666521118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666521118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666521121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666521122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741837_1013 (size=12001) 2024-11-27T00:14:21,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/458c746dbf6d459390093caf01561845 2024-11-27T00:14:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T00:14:21,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666521275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666521276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666521276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666521277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666521277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T00:14:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:21,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/110c1e8804d14ec89790ddd53518dfad is 50, key is test_row_0/B:col10/1732666460969/Put/seqid=0 2024-11-27T00:14:21,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741838_1014 (size=12001) 2024-11-27T00:14:21,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/110c1e8804d14ec89790ddd53518dfad 2024-11-27T00:14:21,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/d5100c14d12c499895bcd561c4a3773b is 50, key is test_row_0/C:col10/1732666460969/Put/seqid=0 2024-11-27T00:14:21,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741839_1015 (size=12001) 2024-11-27T00:14:21,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/d5100c14d12c499895bcd561c4a3773b 2024-11-27T00:14:21,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T00:14:21,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:21,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:21,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:21,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:21,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/458c746dbf6d459390093caf01561845 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/458c746dbf6d459390093caf01561845 2024-11-27T00:14:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666521485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666521487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666521490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/458c746dbf6d459390093caf01561845, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T00:14:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666521495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666521496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/110c1e8804d14ec89790ddd53518dfad as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/110c1e8804d14ec89790ddd53518dfad 2024-11-27T00:14:21,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/110c1e8804d14ec89790ddd53518dfad, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T00:14:21,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/d5100c14d12c499895bcd561c4a3773b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d5100c14d12c499895bcd561c4a3773b 2024-11-27T00:14:21,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d5100c14d12c499895bcd561c4a3773b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T00:14:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T00:14:21,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e7356b094232ddb421ff69892486ac78 in 585ms, sequenceid=13, compaction requested=false 2024-11-27T00:14:21,568 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T00:14:21,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:21,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T00:14:21,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:21,630 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:14:21,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:21,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:21,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:21,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:21,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/122285784d0349b6ad9b5c16fe4b6f32 is 50, key is test_row_0/A:col10/1732666461109/Put/seqid=0 2024-11-27T00:14:21,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741840_1016 (size=12001) 2024-11-27T00:14:21,680 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/122285784d0349b6ad9b5c16fe4b6f32 2024-11-27T00:14:21,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3fc2ae03ecad437f8dfc8aad872e5e8a is 50, key is test_row_0/B:col10/1732666461109/Put/seqid=0 2024-11-27T00:14:21,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741841_1017 (size=12001) 2024-11-27T00:14:21,725 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3fc2ae03ecad437f8dfc8aad872e5e8a 2024-11-27T00:14:21,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/399f14ce91954ff7a3f80de0dcb6cdd4 is 50, key is test_row_0/C:col10/1732666461109/Put/seqid=0 2024-11-27T00:14:21,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741842_1018 (size=12001) 2024-11-27T00:14:21,774 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/399f14ce91954ff7a3f80de0dcb6cdd4 2024-11-27T00:14:21,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/122285784d0349b6ad9b5c16fe4b6f32 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/122285784d0349b6ad9b5c16fe4b6f32 2024-11-27T00:14:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:21,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:21,829 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/122285784d0349b6ad9b5c16fe4b6f32, entries=150, sequenceid=33, filesize=11.7 K 2024-11-27T00:14:21,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3fc2ae03ecad437f8dfc8aad872e5e8a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3fc2ae03ecad437f8dfc8aad872e5e8a 2024-11-27T00:14:21,852 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3fc2ae03ecad437f8dfc8aad872e5e8a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-27T00:14:21,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/399f14ce91954ff7a3f80de0dcb6cdd4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/399f14ce91954ff7a3f80de0dcb6cdd4 2024-11-27T00:14:21,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666521840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666521845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666521850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666521853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666521861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:21,876 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/399f14ce91954ff7a3f80de0dcb6cdd4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-27T00:14:21,878 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e7356b094232ddb421ff69892486ac78 in 248ms, sequenceid=33, compaction requested=false 2024-11-27T00:14:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:21,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-27T00:14:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-27T00:14:21,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T00:14:21,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 963 msec 2024-11-27T00:14:21,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0000 sec 2024-11-27T00:14:21,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T00:14:21,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:21,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:21,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:21,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:21,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:21,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:21,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/120142262ea148b7bd7d62757eb3c19a is 50, key is test_row_0/A:col10/1732666461966/Put/seqid=0 2024-11-27T00:14:22,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741843_1019 (size=12001) 2024-11-27T00:14:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T00:14:22,065 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:22,066 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-27T00:14:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T00:14:22,069 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:22,071 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:22,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:22,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666522082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666522085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666522088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666522088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666522097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T00:14:22,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666522205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666522205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666522206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666522206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666522215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T00:14:22,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:22,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T00:14:22,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T00:14:22,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:22,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666522415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/120142262ea148b7bd7d62757eb3c19a 2024-11-27T00:14:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666522415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666522416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666522416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666522428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/74804aed5ca44fa68eee29f6590600aa is 50, key is test_row_0/B:col10/1732666461966/Put/seqid=0 2024-11-27T00:14:22,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741844_1020 (size=12001) 2024-11-27T00:14:22,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/74804aed5ca44fa68eee29f6590600aa 2024-11-27T00:14:22,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/8625d4e9b64f4875a79a684d8aebf4b9 is 50, key is test_row_0/C:col10/1732666461966/Put/seqid=0 2024-11-27T00:14:22,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T00:14:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741845_1021 (size=12001) 2024-11-27T00:14:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T00:14:22,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T00:14:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:22,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666522725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666522726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666522726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666522734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666522745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:22,820 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T00:14:22,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T00:14:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:22,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:22,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/8625d4e9b64f4875a79a684d8aebf4b9 2024-11-27T00:14:22,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/120142262ea148b7bd7d62757eb3c19a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/120142262ea148b7bd7d62757eb3c19a 2024-11-27T00:14:22,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/120142262ea148b7bd7d62757eb3c19a, entries=150, sequenceid=46, filesize=11.7 K 2024-11-27T00:14:22,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/74804aed5ca44fa68eee29f6590600aa as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/74804aed5ca44fa68eee29f6590600aa 2024-11-27T00:14:23,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T00:14:23,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:23,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:23,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:23,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:23,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:23,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/74804aed5ca44fa68eee29f6590600aa, entries=150, sequenceid=46, filesize=11.7 K 2024-11-27T00:14:23,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/8625d4e9b64f4875a79a684d8aebf4b9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/8625d4e9b64f4875a79a684d8aebf4b9 2024-11-27T00:14:23,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/8625d4e9b64f4875a79a684d8aebf4b9, entries=150, sequenceid=46, filesize=11.7 K 2024-11-27T00:14:23,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e7356b094232ddb421ff69892486ac78 in 1065ms, sequenceid=46, compaction requested=true 2024-11-27T00:14:23,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:23,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:23,040 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:23,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:23,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:23,040 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:23,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:23,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:23,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:23,045 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:23,045 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:23,047 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:23,047 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:23,047 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:23,047 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:23,048 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/458c746dbf6d459390093caf01561845, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/122285784d0349b6ad9b5c16fe4b6f32, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/120142262ea148b7bd7d62757eb3c19a] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=35.2 K 2024-11-27T00:14:23,048 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/110c1e8804d14ec89790ddd53518dfad, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3fc2ae03ecad437f8dfc8aad872e5e8a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/74804aed5ca44fa68eee29f6590600aa] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=35.2 K 2024-11-27T00:14:23,049 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 458c746dbf6d459390093caf01561845, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732666460944 2024-11-27T00:14:23,049 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 110c1e8804d14ec89790ddd53518dfad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732666460944 2024-11-27T00:14:23,050 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fc2ae03ecad437f8dfc8aad872e5e8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732666461108 2024-11-27T00:14:23,050 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 122285784d0349b6ad9b5c16fe4b6f32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732666461108 2024-11-27T00:14:23,051 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 74804aed5ca44fa68eee29f6590600aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666461840 2024-11-27T00:14:23,051 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 120142262ea148b7bd7d62757eb3c19a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666461840 2024-11-27T00:14:23,093 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#9 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:23,094 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/c3874a884ca94ce8b4ee91973a9b9194 is 50, key is test_row_0/A:col10/1732666461966/Put/seqid=0 2024-11-27T00:14:23,102 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#10 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:23,103 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/a36acf4dfd6543abb11903e947918470 is 50, key is test_row_0/B:col10/1732666461966/Put/seqid=0 2024-11-27T00:14:23,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741847_1023 (size=12104) 2024-11-27T00:14:23,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741846_1022 (size=12104) 2024-11-27T00:14:23,154 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/a36acf4dfd6543abb11903e947918470 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a36acf4dfd6543abb11903e947918470 2024-11-27T00:14:23,164 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/c3874a884ca94ce8b4ee91973a9b9194 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c3874a884ca94ce8b4ee91973a9b9194 2024-11-27T00:14:23,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T00:14:23,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:23,168 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:14:23,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:23,192 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into c3874a884ca94ce8b4ee91973a9b9194(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:23,192 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into a36acf4dfd6543abb11903e947918470(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a34835943ed5404a931dae76cb67fc12 is 50, key is test_row_0/A:col10/1732666462091/Put/seqid=0 2024-11-27T00:14:23,192 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:23,193 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:23,196 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=13, startTime=1732666463036; duration=0sec 2024-11-27T00:14:23,196 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=13, startTime=1732666463040; duration=0sec 2024-11-27T00:14:23,196 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:23,196 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:23,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:23,197 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:23,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:23,199 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:23,199 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:23,199 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:23,200 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d5100c14d12c499895bcd561c4a3773b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/399f14ce91954ff7a3f80de0dcb6cdd4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/8625d4e9b64f4875a79a684d8aebf4b9] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=35.2 K 2024-11-27T00:14:23,201 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d5100c14d12c499895bcd561c4a3773b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732666460944 2024-11-27T00:14:23,202 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 399f14ce91954ff7a3f80de0dcb6cdd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732666461108 2024-11-27T00:14:23,203 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8625d4e9b64f4875a79a684d8aebf4b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666461840 2024-11-27T00:14:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T00:14:23,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741848_1024 (size=12001) 2024-11-27T00:14:23,207 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a34835943ed5404a931dae76cb67fc12 2024-11-27T00:14:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:23,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:23,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c364a71e5c3942fdb1feb6d697e9bbe0 is 50, key is test_row_0/B:col10/1732666462091/Put/seqid=0 2024-11-27T00:14:23,283 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:23,284 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/90f6a28765054f4f81b89226a2a3e152 is 50, key is test_row_0/C:col10/1732666461966/Put/seqid=0 2024-11-27T00:14:23,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741849_1025 (size=12104) 2024-11-27T00:14:23,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666523300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741850_1026 (size=12001) 2024-11-27T00:14:23,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,314 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/90f6a28765054f4f81b89226a2a3e152 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/90f6a28765054f4f81b89226a2a3e152 2024-11-27T00:14:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666523300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,316 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c364a71e5c3942fdb1feb6d697e9bbe0 2024-11-27T00:14:23,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666523303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666523307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666523310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,339 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into 90f6a28765054f4f81b89226a2a3e152(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:23,339 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:23,339 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=13, startTime=1732666463041; duration=0sec 2024-11-27T00:14:23,340 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:23,340 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/776b3acc8e4e424c9451ab7bc3e7ef11 is 50, key is test_row_0/C:col10/1732666462091/Put/seqid=0 2024-11-27T00:14:23,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741851_1027 (size=12001) 2024-11-27T00:14:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666523414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666523424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666523425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666523425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666523434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666523624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,627 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T00:14:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666523634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666523635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666523636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666523654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:23,716 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T00:14:23,716 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T00:14:23,719 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T00:14:23,719 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T00:14:23,719 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T00:14:23,720 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T00:14:23,721 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T00:14:23,721 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T00:14:23,765 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/776b3acc8e4e424c9451ab7bc3e7ef11 2024-11-27T00:14:23,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a34835943ed5404a931dae76cb67fc12 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a34835943ed5404a931dae76cb67fc12 2024-11-27T00:14:23,798 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a34835943ed5404a931dae76cb67fc12, entries=150, sequenceid=65, filesize=11.7 K 2024-11-27T00:14:23,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c364a71e5c3942fdb1feb6d697e9bbe0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c364a71e5c3942fdb1feb6d697e9bbe0 2024-11-27T00:14:23,813 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c364a71e5c3942fdb1feb6d697e9bbe0, entries=150, sequenceid=65, filesize=11.7 K 2024-11-27T00:14:23,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/776b3acc8e4e424c9451ab7bc3e7ef11 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/776b3acc8e4e424c9451ab7bc3e7ef11 2024-11-27T00:14:23,845 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/776b3acc8e4e424c9451ab7bc3e7ef11, entries=150, sequenceid=65, filesize=11.7 K 2024-11-27T00:14:23,847 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e7356b094232ddb421ff69892486ac78 in 679ms, sequenceid=65, compaction requested=false 2024-11-27T00:14:23,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:23,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:23,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-27T00:14:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-27T00:14:23,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-27T00:14:23,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7790 sec 2024-11-27T00:14:23,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7920 sec 2024-11-27T00:14:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:23,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:23,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:23,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:23,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:23,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:23,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:23,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:23,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/d98424d7041c49898fd506e69c961738 is 50, key is test_row_0/A:col10/1732666463298/Put/seqid=0 2024-11-27T00:14:23,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741852_1028 (size=16681) 2024-11-27T00:14:23,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/d98424d7041c49898fd506e69c961738 2024-11-27T00:14:24,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ae080615910d4b2783690bcdb814746e is 50, key is test_row_0/B:col10/1732666463298/Put/seqid=0 2024-11-27T00:14:24,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741853_1029 (size=12001) 2024-11-27T00:14:24,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666523998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666524026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666524026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666524030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666524029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666524134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666524145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666524145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666524145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666524145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T00:14:24,214 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:24,215 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-27T00:14:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T00:14:24,218 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:24,219 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:24,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T00:14:24,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666524345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666524354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666524354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666524354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666524365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ae080615910d4b2783690bcdb814746e 2024-11-27T00:14:24,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/2b657ce194594e3b94dde45d685e3044 is 50, key is test_row_0/C:col10/1732666463298/Put/seqid=0 2024-11-27T00:14:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741854_1030 (size=12001) 2024-11-27T00:14:24,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/2b657ce194594e3b94dde45d685e3044 2024-11-27T00:14:24,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/d98424d7041c49898fd506e69c961738 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d98424d7041c49898fd506e69c961738 2024-11-27T00:14:24,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d98424d7041c49898fd506e69c961738, entries=250, sequenceid=83, filesize=16.3 K 2024-11-27T00:14:24,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ae080615910d4b2783690bcdb814746e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ae080615910d4b2783690bcdb814746e 2024-11-27T00:14:24,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ae080615910d4b2783690bcdb814746e, entries=150, sequenceid=83, filesize=11.7 K 2024-11-27T00:14:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/2b657ce194594e3b94dde45d685e3044 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2b657ce194594e3b94dde45d685e3044 2024-11-27T00:14:24,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2b657ce194594e3b94dde45d685e3044, entries=150, sequenceid=83, filesize=11.7 K 2024-11-27T00:14:24,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:24,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:24,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e7356b094232ddb421ff69892486ac78 in 581ms, sequenceid=83, compaction requested=true 2024-11-27T00:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:24,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:24,531 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:24,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,531 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,533 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:24,533 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:24,533 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:24,533 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:24,533 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,533 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,533 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a36acf4dfd6543abb11903e947918470, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c364a71e5c3942fdb1feb6d697e9bbe0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ae080615910d4b2783690bcdb814746e] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=35.3 K 2024-11-27T00:14:24,533 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c3874a884ca94ce8b4ee91973a9b9194, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a34835943ed5404a931dae76cb67fc12, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d98424d7041c49898fd506e69c961738] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=39.8 K 2024-11-27T00:14:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T00:14:24,534 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3874a884ca94ce8b4ee91973a9b9194, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666461840 2024-11-27T00:14:24,534 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting a36acf4dfd6543abb11903e947918470, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666461840 2024-11-27T00:14:24,535 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a34835943ed5404a931dae76cb67fc12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732666462001 2024-11-27T00:14:24,535 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c364a71e5c3942fdb1feb6d697e9bbe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732666462001 2024-11-27T00:14:24,535 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d98424d7041c49898fd506e69c961738, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732666463289 2024-11-27T00:14:24,537 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ae080615910d4b2783690bcdb814746e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732666463298 2024-11-27T00:14:24,557 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:24,558 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/e7508018554f4fe3b32060c75b64f11a is 50, key is test_row_0/B:col10/1732666463298/Put/seqid=0 2024-11-27T00:14:24,558 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:24,560 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/e06f6f0214134b15b89781194b03f9ef is 50, key is test_row_0/A:col10/1732666463298/Put/seqid=0 2024-11-27T00:14:24,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741856_1032 (size=12207) 2024-11-27T00:14:24,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741855_1031 (size=12207) 2024-11-27T00:14:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:24,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:14:24,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:24,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/07e6e54670a443dd9d9378abd59014f8 is 50, key is test_row_0/A:col10/1732666464656/Put/seqid=0 2024-11-27T00:14:24,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:24,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741857_1033 (size=12001) 2024-11-27T00:14:24,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/07e6e54670a443dd9d9378abd59014f8 2024-11-27T00:14:24,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666524686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666524694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/f2054bfb64d241e0a866f55bf1233e8f is 50, key is test_row_0/B:col10/1732666464656/Put/seqid=0 2024-11-27T00:14:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666524697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666524699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666524699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741858_1034 (size=12001) 2024-11-27T00:14:24,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666524804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666524814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666524824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666524825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666524825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:24,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:24,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:24,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T00:14:24,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:24,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:24,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:24,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666525014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,015 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/e06f6f0214134b15b89781194b03f9ef as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e06f6f0214134b15b89781194b03f9ef 2024-11-27T00:14:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,016 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/e7508018554f4fe3b32060c75b64f11a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e7508018554f4fe3b32060c75b64f11a 2024-11-27T00:14:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666525035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666525035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666525035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666525036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,053 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into e06f6f0214134b15b89781194b03f9ef(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:25,053 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into e7508018554f4fe3b32060c75b64f11a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:25,053 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:25,053 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=13, startTime=1732666464531; duration=0sec 2024-11-27T00:14:25,053 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:25,053 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=13, startTime=1732666464531; duration=0sec 2024-11-27T00:14:25,053 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:25,053 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:25,054 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:25,054 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:25,054 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:25,057 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:25,057 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:25,057 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:25,057 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/90f6a28765054f4f81b89226a2a3e152, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/776b3acc8e4e424c9451ab7bc3e7ef11, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2b657ce194594e3b94dde45d685e3044] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=35.3 K 2024-11-27T00:14:25,058 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f6a28765054f4f81b89226a2a3e152, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666461840 2024-11-27T00:14:25,058 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 776b3acc8e4e424c9451ab7bc3e7ef11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732666462001 2024-11-27T00:14:25,059 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b657ce194594e3b94dde45d685e3044, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732666463298 2024-11-27T00:14:25,088 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:25,089 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/48faf2b3428b45f6a474e493a113178a is 50, key is test_row_0/C:col10/1732666463298/Put/seqid=0 2024-11-27T00:14:25,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741859_1035 (size=12207) 2024-11-27T00:14:25,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/f2054bfb64d241e0a866f55bf1233e8f 2024-11-27T00:14:25,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/503760cb334f4063ad71c0012755063d is 50, key is test_row_0/C:col10/1732666464656/Put/seqid=0 2024-11-27T00:14:25,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741860_1036 (size=12001) 2024-11-27T00:14:25,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:25,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:25,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:25,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:25,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:25,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:25,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:25,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666525324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T00:14:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666525355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666525356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666525357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666525357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:25,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:25,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:25,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:25,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:25,519 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/48faf2b3428b45f6a474e493a113178a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48faf2b3428b45f6a474e493a113178a 2024-11-27T00:14:25,532 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into 48faf2b3428b45f6a474e493a113178a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:25,532 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:25,533 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=13, startTime=1732666464531; duration=0sec 2024-11-27T00:14:25,533 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:25,533 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:25,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/503760cb334f4063ad71c0012755063d 2024-11-27T00:14:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/07e6e54670a443dd9d9378abd59014f8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/07e6e54670a443dd9d9378abd59014f8 2024-11-27T00:14:25,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/07e6e54670a443dd9d9378abd59014f8, entries=150, sequenceid=102, filesize=11.7 K 2024-11-27T00:14:25,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/f2054bfb64d241e0a866f55bf1233e8f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f2054bfb64d241e0a866f55bf1233e8f 2024-11-27T00:14:25,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f2054bfb64d241e0a866f55bf1233e8f, entries=150, sequenceid=102, filesize=11.7 K 2024-11-27T00:14:25,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/503760cb334f4063ad71c0012755063d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/503760cb334f4063ad71c0012755063d 2024-11-27T00:14:25,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/503760cb334f4063ad71c0012755063d, entries=150, sequenceid=102, filesize=11.7 K 2024-11-27T00:14:25,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e7356b094232ddb421ff69892486ac78 in 953ms, sequenceid=102, compaction requested=false 2024-11-27T00:14:25,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:25,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T00:14:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:25,617 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:25,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/964f7e27a6444a779de759fb29b77175 is 50, key is test_row_0/A:col10/1732666464696/Put/seqid=0 2024-11-27T00:14:25,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741861_1037 (size=12001) 2024-11-27T00:14:25,646 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/964f7e27a6444a779de759fb29b77175 2024-11-27T00:14:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3e22d940fd344bf496ac91cacbe960f6 is 50, key is test_row_0/B:col10/1732666464696/Put/seqid=0 2024-11-27T00:14:25,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741862_1038 (size=12001) 2024-11-27T00:14:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:25,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:25,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666525877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666525878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666525880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666525882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666525882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666525994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666525994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666525994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:25,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666525995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666525995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:25,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:25,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,072 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3e22d940fd344bf496ac91cacbe960f6 2024-11-27T00:14:26,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/2f450cc7d86d430888a95929ea7648b2 is 50, key is test_row_0/C:col10/1732666464696/Put/seqid=0 2024-11-27T00:14:26,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741863_1039 (size=12001) 2024-11-27T00:14:26,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666526203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666526204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666526204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666526204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666526204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T00:14:26,490 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/2f450cc7d86d430888a95929ea7648b2 2024-11-27T00:14:26,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/964f7e27a6444a779de759fb29b77175 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/964f7e27a6444a779de759fb29b77175 2024-11-27T00:14:26,505 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/964f7e27a6444a779de759fb29b77175, entries=150, sequenceid=119, filesize=11.7 K 2024-11-27T00:14:26,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3e22d940fd344bf496ac91cacbe960f6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e22d940fd344bf496ac91cacbe960f6 2024-11-27T00:14:26,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666526515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666526515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666526515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666526515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666526515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:26,520 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e22d940fd344bf496ac91cacbe960f6, entries=150, sequenceid=119, filesize=11.7 K 2024-11-27T00:14:26,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/2f450cc7d86d430888a95929ea7648b2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2f450cc7d86d430888a95929ea7648b2 2024-11-27T00:14:26,535 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2f450cc7d86d430888a95929ea7648b2, entries=150, sequenceid=119, filesize=11.7 K 2024-11-27T00:14:26,537 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e7356b094232ddb421ff69892486ac78 in 920ms, sequenceid=119, compaction requested=true 2024-11-27T00:14:26,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:26,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:26,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-27T00:14:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-27T00:14:26,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T00:14:26,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-11-27T00:14:26,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.3320 sec 2024-11-27T00:14:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:27,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:14:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:27,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:27,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:27,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:27,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:27,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:27,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b3c854faf2054d6b83559119398e4daa is 50, key is test_row_0/A:col10/1732666467026/Put/seqid=0 2024-11-27T00:14:27,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666527040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666527039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666527044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666527045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666527052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741864_1040 (size=14541) 2024-11-27T00:14:27,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666527154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666527164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666527165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666527165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666527165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666527364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666527375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666527376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666527376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666527376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b3c854faf2054d6b83559119398e4daa 2024-11-27T00:14:27,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/40f853961b6e45e2bb23f9a4d880e54d is 50, key is test_row_0/B:col10/1732666467026/Put/seqid=0 2024-11-27T00:14:27,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741865_1041 (size=12151) 2024-11-27T00:14:27,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/40f853961b6e45e2bb23f9a4d880e54d 2024-11-27T00:14:27,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/fb00401af284458aa5a02a7b54cdbedf is 50, key is test_row_0/C:col10/1732666467026/Put/seqid=0 2024-11-27T00:14:27,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741866_1042 (size=12151) 2024-11-27T00:14:27,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/fb00401af284458aa5a02a7b54cdbedf 2024-11-27T00:14:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b3c854faf2054d6b83559119398e4daa as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b3c854faf2054d6b83559119398e4daa 2024-11-27T00:14:27,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b3c854faf2054d6b83559119398e4daa, entries=200, sequenceid=138, filesize=14.2 K 2024-11-27T00:14:27,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/40f853961b6e45e2bb23f9a4d880e54d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/40f853961b6e45e2bb23f9a4d880e54d 2024-11-27T00:14:27,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/40f853961b6e45e2bb23f9a4d880e54d, entries=150, sequenceid=138, filesize=11.9 K 2024-11-27T00:14:27,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/fb00401af284458aa5a02a7b54cdbedf as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/fb00401af284458aa5a02a7b54cdbedf 2024-11-27T00:14:27,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/fb00401af284458aa5a02a7b54cdbedf, entries=150, sequenceid=138, filesize=11.9 K 2024-11-27T00:14:27,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e7356b094232ddb421ff69892486ac78 in 558ms, sequenceid=138, compaction requested=true 2024-11-27T00:14:27,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:27,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:27,587 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:27,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:27,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:27,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:27,587 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:27,590 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:27,590 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:27,590 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:27,590 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e06f6f0214134b15b89781194b03f9ef, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/07e6e54670a443dd9d9378abd59014f8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/964f7e27a6444a779de759fb29b77175, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b3c854faf2054d6b83559119398e4daa] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=49.6 K 2024-11-27T00:14:27,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:27,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:27,590 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:27,590 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:27,591 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:27,591 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e7508018554f4fe3b32060c75b64f11a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f2054bfb64d241e0a866f55bf1233e8f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e22d940fd344bf496ac91cacbe960f6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/40f853961b6e45e2bb23f9a4d880e54d] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=47.2 K 2024-11-27T00:14:27,591 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e7508018554f4fe3b32060c75b64f11a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732666463298 2024-11-27T00:14:27,591 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06f6f0214134b15b89781194b03f9ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732666463298 2024-11-27T00:14:27,592 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e6e54670a443dd9d9378abd59014f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732666463986 2024-11-27T00:14:27,592 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f2054bfb64d241e0a866f55bf1233e8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732666463986 2024-11-27T00:14:27,593 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 964f7e27a6444a779de759fb29b77175, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732666464685 2024-11-27T00:14:27,593 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e22d940fd344bf496ac91cacbe960f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732666464685 2024-11-27T00:14:27,594 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3c854faf2054d6b83559119398e4daa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732666465871 2024-11-27T00:14:27,594 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f853961b6e45e2bb23f9a4d880e54d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732666465871 2024-11-27T00:14:27,621 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:27,622 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/24b363f3390f4473a3431ffae6c3375f is 50, key is test_row_0/A:col10/1732666467026/Put/seqid=0 2024-11-27T00:14:27,625 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:27,626 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/043c03542fa24da9b2418520eac770ce is 50, key is test_row_0/B:col10/1732666467026/Put/seqid=0 2024-11-27T00:14:27,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741867_1043 (size=12493) 2024-11-27T00:14:27,663 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/24b363f3390f4473a3431ffae6c3375f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/24b363f3390f4473a3431ffae6c3375f 2024-11-27T00:14:27,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741868_1044 (size=12493) 2024-11-27T00:14:27,677 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into 24b363f3390f4473a3431ffae6c3375f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:27,678 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:27,678 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=12, startTime=1732666467587; duration=0sec 2024-11-27T00:14:27,678 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:27,678 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:27,681 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:27,682 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/043c03542fa24da9b2418520eac770ce as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/043c03542fa24da9b2418520eac770ce 2024-11-27T00:14:27,683 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:27,683 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:27,683 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:27,683 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48faf2b3428b45f6a474e493a113178a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/503760cb334f4063ad71c0012755063d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2f450cc7d86d430888a95929ea7648b2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/fb00401af284458aa5a02a7b54cdbedf] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=47.2 K 2024-11-27T00:14:27,687 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48faf2b3428b45f6a474e493a113178a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732666463298 2024-11-27T00:14:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:27,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:27,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:27,689 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503760cb334f4063ad71c0012755063d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732666463986 2024-11-27T00:14:27,692 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f450cc7d86d430888a95929ea7648b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732666464685 2024-11-27T00:14:27,694 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb00401af284458aa5a02a7b54cdbedf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732666465871 2024-11-27T00:14:27,696 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into 043c03542fa24da9b2418520eac770ce(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:27,696 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:27,696 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=12, startTime=1732666467587; duration=0sec 2024-11-27T00:14:27,697 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:27,697 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:27,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0084dbb3f42e40fdafb6720afa77be4f is 50, key is test_row_0/A:col10/1732666467039/Put/seqid=0 2024-11-27T00:14:27,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741869_1045 (size=12151) 2024-11-27T00:14:27,723 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:27,724 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/48a84a22dba34a48b87b10a10159883a is 50, key is test_row_0/C:col10/1732666467026/Put/seqid=0 2024-11-27T00:14:27,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0084dbb3f42e40fdafb6720afa77be4f 2024-11-27T00:14:27,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666527725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666527729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666527730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666527733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666527734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741870_1046 (size=12493) 2024-11-27T00:14:27,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ad5178f325df47ff93c80561b9acace9 is 50, key is test_row_0/B:col10/1732666467039/Put/seqid=0 2024-11-27T00:14:27,772 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/48a84a22dba34a48b87b10a10159883a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48a84a22dba34a48b87b10a10159883a 2024-11-27T00:14:27,787 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into 48a84a22dba34a48b87b10a10159883a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:27,787 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:27,787 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=12, startTime=1732666467587; duration=0sec 2024-11-27T00:14:27,788 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:27,788 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:27,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741871_1047 (size=12151) 2024-11-27T00:14:27,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ad5178f325df47ff93c80561b9acace9 2024-11-27T00:14:27,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/d0b7a206e7c74c20870e639b34148307 is 50, key is test_row_0/C:col10/1732666467039/Put/seqid=0 2024-11-27T00:14:27,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741872_1048 (size=12151) 2024-11-27T00:14:27,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/d0b7a206e7c74c20870e639b34148307 2024-11-27T00:14:27,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0084dbb3f42e40fdafb6720afa77be4f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0084dbb3f42e40fdafb6720afa77be4f 2024-11-27T00:14:27,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666527845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666527845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666527845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666527854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666527855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:27,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0084dbb3f42e40fdafb6720afa77be4f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T00:14:27,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ad5178f325df47ff93c80561b9acace9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ad5178f325df47ff93c80561b9acace9 2024-11-27T00:14:27,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ad5178f325df47ff93c80561b9acace9, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T00:14:27,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/d0b7a206e7c74c20870e639b34148307 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d0b7a206e7c74c20870e639b34148307 2024-11-27T00:14:27,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d0b7a206e7c74c20870e639b34148307, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T00:14:27,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e7356b094232ddb421ff69892486ac78 in 209ms, sequenceid=153, compaction requested=false 2024-11-27T00:14:27,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:28,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:14:28,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:28,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:28,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:28,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:28,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:28,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:28,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/4729a3d3384b4234be73f58ae6765894 is 50, key is test_row_0/A:col10/1732666467727/Put/seqid=0 2024-11-27T00:14:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666528079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666528080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741873_1049 (size=12151) 2024-11-27T00:14:28,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666528084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666528085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666528085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666528194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666528194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666528194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666528194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666528195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T00:14:28,395 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:28,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-27T00:14:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T00:14:28,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:28,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:28,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666528404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666528404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666528404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666528404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666528404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/4729a3d3384b4234be73f58ae6765894 2024-11-27T00:14:28,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/acef2d147da247319338c2cea6479e15 is 50, key is test_row_0/B:col10/1732666467727/Put/seqid=0 2024-11-27T00:14:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T00:14:28,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741874_1050 (size=12151) 2024-11-27T00:14:28,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/acef2d147da247319338c2cea6479e15 2024-11-27T00:14:28,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/36a0e9aa520e4490a99d229e69c0acbe is 50, key is test_row_0/C:col10/1732666467727/Put/seqid=0 2024-11-27T00:14:28,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T00:14:28,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:28,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:28,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:28,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:28,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:28,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741875_1051 (size=12151) 2024-11-27T00:14:28,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/36a0e9aa520e4490a99d229e69c0acbe 2024-11-27T00:14:28,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/4729a3d3384b4234be73f58ae6765894 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4729a3d3384b4234be73f58ae6765894 2024-11-27T00:14:28,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4729a3d3384b4234be73f58ae6765894, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T00:14:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/acef2d147da247319338c2cea6479e15 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/acef2d147da247319338c2cea6479e15 2024-11-27T00:14:28,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/acef2d147da247319338c2cea6479e15, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T00:14:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/36a0e9aa520e4490a99d229e69c0acbe as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/36a0e9aa520e4490a99d229e69c0acbe 2024-11-27T00:14:28,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/36a0e9aa520e4490a99d229e69c0acbe, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T00:14:28,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e7356b094232ddb421ff69892486ac78 in 553ms, sequenceid=174, compaction requested=true 2024-11-27T00:14:28,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:28,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:28,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:28,613 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:28,613 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:28,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:28,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:28,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:28,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:28,614 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:28,614 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:28,614 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:28,615 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/043c03542fa24da9b2418520eac770ce, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ad5178f325df47ff93c80561b9acace9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/acef2d147da247319338c2cea6479e15] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=35.9 K 2024-11-27T00:14:28,615 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 043c03542fa24da9b2418520eac770ce, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732666465871 2024-11-27T00:14:28,616 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:28,616 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:28,616 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:28,616 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/24b363f3390f4473a3431ffae6c3375f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0084dbb3f42e40fdafb6720afa77be4f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4729a3d3384b4234be73f58ae6765894] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=35.9 K 2024-11-27T00:14:28,616 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ad5178f325df47ff93c80561b9acace9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732666467039 2024-11-27T00:14:28,617 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b363f3390f4473a3431ffae6c3375f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732666465871 2024-11-27T00:14:28,617 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting acef2d147da247319338c2cea6479e15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666467727 2024-11-27T00:14:28,618 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0084dbb3f42e40fdafb6720afa77be4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732666467039 2024-11-27T00:14:28,619 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4729a3d3384b4234be73f58ae6765894, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666467727 2024-11-27T00:14:28,641 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:28,641 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/d863e3c695a7404ea4b68d76c935e326 is 50, key is test_row_0/B:col10/1732666467727/Put/seqid=0 2024-11-27T00:14:28,642 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:28,643 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/cebe18fc9ea04a94a61caad486c80788 is 50, key is test_row_0/A:col10/1732666467727/Put/seqid=0 2024-11-27T00:14:28,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741876_1052 (size=12595) 2024-11-27T00:14:28,683 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/d863e3c695a7404ea4b68d76c935e326 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/d863e3c695a7404ea4b68d76c935e326 2024-11-27T00:14:28,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T00:14:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:28,711 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:28,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T00:14:28,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741877_1053 (size=12595) 2024-11-27T00:14:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:28,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:28,732 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/cebe18fc9ea04a94a61caad486c80788 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/cebe18fc9ea04a94a61caad486c80788 2024-11-27T00:14:28,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b0e70f30d0784b48a97ea7531b11a673 is 50, key is test_row_0/A:col10/1732666468064/Put/seqid=0 2024-11-27T00:14:28,737 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into d863e3c695a7404ea4b68d76c935e326(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:28,737 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:28,737 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=13, startTime=1732666468613; duration=0sec 2024-11-27T00:14:28,738 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:28,738 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:28,738 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:28,740 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:28,740 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:28,741 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:28,741 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48a84a22dba34a48b87b10a10159883a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d0b7a206e7c74c20870e639b34148307, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/36a0e9aa520e4490a99d229e69c0acbe] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=35.9 K 2024-11-27T00:14:28,742 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 48a84a22dba34a48b87b10a10159883a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732666465871 2024-11-27T00:14:28,742 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d0b7a206e7c74c20870e639b34148307, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732666467039 2024-11-27T00:14:28,743 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a0e9aa520e4490a99d229e69c0acbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666467727 2024-11-27T00:14:28,753 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into cebe18fc9ea04a94a61caad486c80788(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:28,753 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:28,753 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=13, startTime=1732666468613; duration=0sec 2024-11-27T00:14:28,754 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:28,754 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:28,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666528751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,767 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:28,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666528758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666528755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,771 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/27c3cccac11d49cb9ee5617a055926da is 50, key is test_row_0/C:col10/1732666467727/Put/seqid=0 2024-11-27T00:14:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666528761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666528761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741878_1054 (size=12151) 2024-11-27T00:14:28,801 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b0e70f30d0784b48a97ea7531b11a673 2024-11-27T00:14:28,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741879_1055 (size=12595) 2024-11-27T00:14:28,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c345bfab3d1c4c1ca0f229bfd203bb04 is 50, key is test_row_0/B:col10/1732666468064/Put/seqid=0 2024-11-27T00:14:28,831 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/27c3cccac11d49cb9ee5617a055926da as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/27c3cccac11d49cb9ee5617a055926da 2024-11-27T00:14:28,845 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into 27c3cccac11d49cb9ee5617a055926da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:28,845 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:28,847 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=13, startTime=1732666468613; duration=0sec 2024-11-27T00:14:28,847 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:28,847 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:28,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741880_1056 (size=12151) 2024-11-27T00:14:28,864 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c345bfab3d1c4c1ca0f229bfd203bb04 2024-11-27T00:14:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666528864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666528874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666528874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666528875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666528876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:28,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:28,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/28cf16cabc8d40ed8215023caf235099 is 50, key is test_row_0/C:col10/1732666468064/Put/seqid=0 2024-11-27T00:14:28,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741881_1057 (size=12151) 2024-11-27T00:14:28,911 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/28cf16cabc8d40ed8215023caf235099 2024-11-27T00:14:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b0e70f30d0784b48a97ea7531b11a673 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b0e70f30d0784b48a97ea7531b11a673 2024-11-27T00:14:28,932 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b0e70f30d0784b48a97ea7531b11a673, entries=150, sequenceid=188, filesize=11.9 K 2024-11-27T00:14:28,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c345bfab3d1c4c1ca0f229bfd203bb04 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c345bfab3d1c4c1ca0f229bfd203bb04 2024-11-27T00:14:28,944 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c345bfab3d1c4c1ca0f229bfd203bb04, entries=150, sequenceid=188, filesize=11.9 K 2024-11-27T00:14:28,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/28cf16cabc8d40ed8215023caf235099 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/28cf16cabc8d40ed8215023caf235099 2024-11-27T00:14:28,955 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/28cf16cabc8d40ed8215023caf235099, entries=150, sequenceid=188, filesize=11.9 K 2024-11-27T00:14:28,958 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for e7356b094232ddb421ff69892486ac78 in 247ms, sequenceid=188, compaction requested=false 2024-11-27T00:14:28,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:28,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:28,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-27T00:14:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-27T00:14:28,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T00:14:28,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 563 msec 2024-11-27T00:14:28,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 573 msec 2024-11-27T00:14:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T00:14:29,025 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:29,025 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-27T00:14:29,030 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T00:14:29,031 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:29,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:29,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:14:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:29,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/e9575f9dbc7e4ce298cd73f151d98309 is 50, key is test_row_0/A:col10/1732666468758/Put/seqid=0 2024-11-27T00:14:29,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666529094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666529093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666529100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741882_1058 (size=14541) 2024-11-27T00:14:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/e9575f9dbc7e4ce298cd73f151d98309 2024-11-27T00:14:29,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666529105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666529106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/f536d437572b4b63a2c957bccf808e18 is 50, key is test_row_0/B:col10/1732666468758/Put/seqid=0 2024-11-27T00:14:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T00:14:29,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741883_1059 (size=12151) 2024-11-27T00:14:29,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/f536d437572b4b63a2c957bccf808e18 2024-11-27T00:14:29,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/eac04c53b8334726813e74a290459bcd is 50, key is test_row_0/C:col10/1732666468758/Put/seqid=0 2024-11-27T00:14:29,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T00:14:29,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:29,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741884_1060 (size=12151) 2024-11-27T00:14:29,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/eac04c53b8334726813e74a290459bcd 2024-11-27T00:14:29,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/e9575f9dbc7e4ce298cd73f151d98309 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e9575f9dbc7e4ce298cd73f151d98309 2024-11-27T00:14:29,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666529214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666529214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666529215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666529215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666529215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e9575f9dbc7e4ce298cd73f151d98309, entries=200, sequenceid=211, filesize=14.2 K 2024-11-27T00:14:29,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/f536d437572b4b63a2c957bccf808e18 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f536d437572b4b63a2c957bccf808e18 2024-11-27T00:14:29,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f536d437572b4b63a2c957bccf808e18, entries=150, sequenceid=211, filesize=11.9 K 2024-11-27T00:14:29,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/eac04c53b8334726813e74a290459bcd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/eac04c53b8334726813e74a290459bcd 2024-11-27T00:14:29,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/eac04c53b8334726813e74a290459bcd, entries=150, sequenceid=211, filesize=11.9 K 2024-11-27T00:14:29,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e7356b094232ddb421ff69892486ac78 in 177ms, sequenceid=211, compaction requested=true 2024-11-27T00:14:29,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:29,257 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:29,257 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:29,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:29,258 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:29,259 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:29,259 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,259 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/d863e3c695a7404ea4b68d76c935e326, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c345bfab3d1c4c1ca0f229bfd203bb04, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f536d437572b4b63a2c957bccf808e18] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.0 K 2024-11-27T00:14:29,259 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:29,259 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:29,259 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,260 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/cebe18fc9ea04a94a61caad486c80788, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b0e70f30d0784b48a97ea7531b11a673, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e9575f9dbc7e4ce298cd73f151d98309] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=38.4 K 2024-11-27T00:14:29,261 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d863e3c695a7404ea4b68d76c935e326, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666467727 2024-11-27T00:14:29,261 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting cebe18fc9ea04a94a61caad486c80788, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666467727 2024-11-27T00:14:29,261 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c345bfab3d1c4c1ca0f229bfd203bb04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732666468064 2024-11-27T00:14:29,261 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0e70f30d0784b48a97ea7531b11a673, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732666468064 2024-11-27T00:14:29,262 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9575f9dbc7e4ce298cd73f151d98309, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732666468755 2024-11-27T00:14:29,262 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f536d437572b4b63a2c957bccf808e18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732666468758 2024-11-27T00:14:29,303 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:29,304 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/526721fd5dc5439da338f5ecd5fc3f53 is 50, key is test_row_0/B:col10/1732666468758/Put/seqid=0 2024-11-27T00:14:29,306 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#48 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:29,308 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/ec4f010057a74acd8ce0f20c2f958177 is 50, key is test_row_0/A:col10/1732666468758/Put/seqid=0 2024-11-27T00:14:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T00:14:29,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T00:14:29,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,345 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T00:14:29,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:29,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:29,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:29,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:29,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:29,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:29,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741886_1062 (size=12697) 2024-11-27T00:14:29,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/fa748dbe926d43f2bad4a8b0e50b2ebb is 50, key is test_row_0/A:col10/1732666469090/Put/seqid=0 2024-11-27T00:14:29,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741885_1061 (size=12697) 2024-11-27T00:14:29,367 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/ec4f010057a74acd8ce0f20c2f958177 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/ec4f010057a74acd8ce0f20c2f958177 2024-11-27T00:14:29,376 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into ec4f010057a74acd8ce0f20c2f958177(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:29,376 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:29,377 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=13, startTime=1732666469256; duration=0sec 2024-11-27T00:14:29,377 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:29,377 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:29,377 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:29,379 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:29,380 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:29,380 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,380 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/27c3cccac11d49cb9ee5617a055926da, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/28cf16cabc8d40ed8215023caf235099, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/eac04c53b8334726813e74a290459bcd] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.0 K 2024-11-27T00:14:29,381 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c3cccac11d49cb9ee5617a055926da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666467727 2024-11-27T00:14:29,381 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28cf16cabc8d40ed8215023caf235099, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732666468064 2024-11-27T00:14:29,382 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting eac04c53b8334726813e74a290459bcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732666468758 2024-11-27T00:14:29,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741887_1063 (size=12151) 2024-11-27T00:14:29,393 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/fa748dbe926d43f2bad4a8b0e50b2ebb 2024-11-27T00:14:29,398 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:29,399 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/1504c45d7743403ca45186c49787d819 is 50, key is test_row_0/C:col10/1732666468758/Put/seqid=0 2024-11-27T00:14:29,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741888_1064 (size=12697) 2024-11-27T00:14:29,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/1b628aebcd114616a09b19688a56db6b is 50, key is test_row_0/B:col10/1732666469090/Put/seqid=0 2024-11-27T00:14:29,425 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/1504c45d7743403ca45186c49787d819 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1504c45d7743403ca45186c49787d819 2024-11-27T00:14:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:29,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:29,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741889_1065 (size=12151) 2024-11-27T00:14:29,434 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/1b628aebcd114616a09b19688a56db6b 2024-11-27T00:14:29,436 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into 1504c45d7743403ca45186c49787d819(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:29,436 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:29,436 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=13, startTime=1732666469257; duration=0sec 2024-11-27T00:14:29,436 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:29,436 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:29,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/cd0067ee4ebf4cd7b91b93d193adb5b6 is 50, key is test_row_0/C:col10/1732666469090/Put/seqid=0 2024-11-27T00:14:29,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741890_1066 (size=12151) 2024-11-27T00:14:29,460 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/cd0067ee4ebf4cd7b91b93d193adb5b6 2024-11-27T00:14:29,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666529457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666529458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666529459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/fa748dbe926d43f2bad4a8b0e50b2ebb as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fa748dbe926d43f2bad4a8b0e50b2ebb 2024-11-27T00:14:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666529459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666529464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,486 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fa748dbe926d43f2bad4a8b0e50b2ebb, entries=150, sequenceid=223, filesize=11.9 K 2024-11-27T00:14:29,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/1b628aebcd114616a09b19688a56db6b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1b628aebcd114616a09b19688a56db6b 2024-11-27T00:14:29,506 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1b628aebcd114616a09b19688a56db6b, entries=150, sequenceid=223, filesize=11.9 K 2024-11-27T00:14:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/cd0067ee4ebf4cd7b91b93d193adb5b6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/cd0067ee4ebf4cd7b91b93d193adb5b6 2024-11-27T00:14:29,527 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/cd0067ee4ebf4cd7b91b93d193adb5b6, entries=150, sequenceid=223, filesize=11.9 K 2024-11-27T00:14:29,530 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e7356b094232ddb421ff69892486ac78 in 185ms, sequenceid=223, compaction requested=false 2024-11-27T00:14:29,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:29,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-27T00:14:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-27T00:14:29,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-27T00:14:29,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 502 msec 2024-11-27T00:14:29,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 512 msec 2024-11-27T00:14:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:29,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-27T00:14:29,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:29,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666469455, max timestamp=1732666469576Num uniques 859; 2024-11-27T00:14:29,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666469455, max timestamp=1732666469576Num uniques 859; 2024-11-27T00:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:29,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:14:29,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:29,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:29,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:14:29,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/829ad45265344824b6e03bb0cd0b71ee is 50, key is test_row_0/A:col10/1732666469582/Put/seqid=0 2024-11-27T00:14:29,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741891_1067 (size=12151) 2024-11-27T00:14:29,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/829ad45265344824b6e03bb0cd0b71ee 2024-11-27T00:14:29,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666529630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666529630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666529633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666529633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666529635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/743edfd38b1241a3a91d1d2af03495c5 is 50, key is test_row_0/B:col10/1732666469582/Put/seqid=0 2024-11-27T00:14:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T00:14:29,654 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:29,655 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-27T00:14:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T00:14:29,658 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:29,660 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:29,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:29,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741892_1068 (size=12151) 2024-11-27T00:14:29,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/743edfd38b1241a3a91d1d2af03495c5 2024-11-27T00:14:29,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/18bf1eed84ea471d8463fc94dfd8f2bd is 50, key is test_row_0/C:col10/1732666469582/Put/seqid=0 2024-11-27T00:14:29,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741893_1069 (size=12151) 2024-11-27T00:14:29,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/18bf1eed84ea471d8463fc94dfd8f2bd 2024-11-27T00:14:29,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/829ad45265344824b6e03bb0cd0b71ee as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/829ad45265344824b6e03bb0cd0b71ee 2024-11-27T00:14:29,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/829ad45265344824b6e03bb0cd0b71ee, entries=150, sequenceid=248, filesize=11.9 K 2024-11-27T00:14:29,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/743edfd38b1241a3a91d1d2af03495c5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/743edfd38b1241a3a91d1d2af03495c5 2024-11-27T00:14:29,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/743edfd38b1241a3a91d1d2af03495c5, entries=150, sequenceid=248, filesize=11.9 K 2024-11-27T00:14:29,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/18bf1eed84ea471d8463fc94dfd8f2bd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/18bf1eed84ea471d8463fc94dfd8f2bd 2024-11-27T00:14:29,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/18bf1eed84ea471d8463fc94dfd8f2bd, entries=150, sequenceid=248, filesize=11.9 K 2024-11-27T00:14:29,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=107.34 KB/109920 for e7356b094232ddb421ff69892486ac78 in 150ms, sequenceid=248, compaction requested=true 2024-11-27T00:14:29,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:29,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:29,735 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:29,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:29,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:29,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:29,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:29,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T00:14:29,736 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:29,736 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:29,736 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,737 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/ec4f010057a74acd8ce0f20c2f958177, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fa748dbe926d43f2bad4a8b0e50b2ebb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/829ad45265344824b6e03bb0cd0b71ee] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.1 K 2024-11-27T00:14:29,738 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec4f010057a74acd8ce0f20c2f958177, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732666468758 2024-11-27T00:14:29,738 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa748dbe926d43f2bad4a8b0e50b2ebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732666469090 2024-11-27T00:14:29,739 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 829ad45265344824b6e03bb0cd0b71ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732666469458 2024-11-27T00:14:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:29,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=114.05 KB heapSize=340.13 KB 2024-11-27T00:14:29,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666469630, max timestamp=1732666469750Num uniques 859; 2024-11-27T00:14:29,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:14:29,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666469630, max timestamp=1732666469750Num uniques 859; 2024-11-27T00:14:29,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:14:29,762 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/526721fd5dc5439da338f5ecd5fc3f53 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/526721fd5dc5439da338f5ecd5fc3f53 2024-11-27T00:14:29,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:29,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:29,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:29,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:29,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:29,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T00:14:29,778 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into 526721fd5dc5439da338f5ecd5fc3f53(size=12.4 K), total size for store is 36.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:29,778 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:29,778 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=13, startTime=1732666469257; duration=0sec 2024-11-27T00:14:29,778 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T00:14:29,778 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:29,779 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:29,779 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:29,779 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:29,780 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a1e50698e74f40a3a8f2eda1fb98b7d8 is 50, key is test_row_0/A:col10/1732666469582/Put/seqid=0 2024-11-27T00:14:29,784 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:29,784 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:29,784 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,784 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1504c45d7743403ca45186c49787d819, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/cd0067ee4ebf4cd7b91b93d193adb5b6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/18bf1eed84ea471d8463fc94dfd8f2bd] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.1 K 2024-11-27T00:14:29,785 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1504c45d7743403ca45186c49787d819, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732666468758 2024-11-27T00:14:29,786 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting cd0067ee4ebf4cd7b91b93d193adb5b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732666469090 2024-11-27T00:14:29,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b99862942db944808153c82712a8cb92 is 50, key is test_row_0/A:col10/1732666469750/Put/seqid=0 2024-11-27T00:14:29,788 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 18bf1eed84ea471d8463fc94dfd8f2bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732666469458 2024-11-27T00:14:29,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666529796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666529799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666529799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666529806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T00:14:29,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:29,814 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:29,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,815 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/6658cfca588c4c519235eb2fd03883eb is 50, key is test_row_0/C:col10/1732666469582/Put/seqid=0 2024-11-27T00:14:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741894_1070 (size=12799) 2024-11-27T00:14:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666529811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741895_1071 (size=14741) 2024-11-27T00:14:29,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b99862942db944808153c82712a8cb92 2024-11-27T00:14:29,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c5f958023dd249f69cd55f7987fed4c5 is 50, key is test_row_0/B:col10/1732666469750/Put/seqid=0 2024-11-27T00:14:29,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741896_1072 (size=12799) 2024-11-27T00:14:29,869 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/6658cfca588c4c519235eb2fd03883eb as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/6658cfca588c4c519235eb2fd03883eb 2024-11-27T00:14:29,882 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into 6658cfca588c4c519235eb2fd03883eb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:29,883 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:29,883 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=13, startTime=1732666469735; duration=0sec 2024-11-27T00:14:29,883 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:29,883 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:29,883 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:29,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741897_1073 (size=12301) 2024-11-27T00:14:29,886 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:29,887 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:29,887 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,888 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/526721fd5dc5439da338f5ecd5fc3f53, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1b628aebcd114616a09b19688a56db6b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/743edfd38b1241a3a91d1d2af03495c5] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.1 K 2024-11-27T00:14:29,888 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 526721fd5dc5439da338f5ecd5fc3f53, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732666468758 2024-11-27T00:14:29,889 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b628aebcd114616a09b19688a56db6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732666469090 2024-11-27T00:14:29,889 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 743edfd38b1241a3a91d1d2af03495c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732666469458 2024-11-27T00:14:29,904 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:29,904 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/38dc10e6fdb44ddc9bc6c41fa6b1155b is 50, key is test_row_0/B:col10/1732666469582/Put/seqid=0 2024-11-27T00:14:29,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666529914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666529914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666529926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666529926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741898_1074 (size=12799) 2024-11-27T00:14:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666529926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:29,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:29,942 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/38dc10e6fdb44ddc9bc6c41fa6b1155b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/38dc10e6fdb44ddc9bc6c41fa6b1155b 2024-11-27T00:14:29,954 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into 38dc10e6fdb44ddc9bc6c41fa6b1155b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:29,954 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:29,954 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=13, startTime=1732666469735; duration=0sec 2024-11-27T00:14:29,954 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:29,954 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:29,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T00:14:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:29,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T00:14:30,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T00:14:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:30,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:30,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666530125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666530125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666530134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666530134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666530144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,225 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a1e50698e74f40a3a8f2eda1fb98b7d8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a1e50698e74f40a3a8f2eda1fb98b7d8 2024-11-27T00:14:30,231 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into a1e50698e74f40a3a8f2eda1fb98b7d8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:30,231 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:30,231 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=13, startTime=1732666469735; duration=0sec 2024-11-27T00:14:30,232 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:30,232 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:30,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T00:14:30,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:30,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:30,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:30,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:30,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T00:14:30,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c5f958023dd249f69cd55f7987fed4c5 2024-11-27T00:14:30,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/ff66630c64874d8a867a8c0377b23a17 is 50, key is test_row_0/C:col10/1732666469750/Put/seqid=0 2024-11-27T00:14:30,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741899_1075 (size=12301) 2024-11-27T00:14:30,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/ff66630c64874d8a867a8c0377b23a17 2024-11-27T00:14:30,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/b99862942db944808153c82712a8cb92 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b99862942db944808153c82712a8cb92 2024-11-27T00:14:30,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b99862942db944808153c82712a8cb92, entries=200, sequenceid=269, filesize=14.4 K 2024-11-27T00:14:30,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/c5f958023dd249f69cd55f7987fed4c5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c5f958023dd249f69cd55f7987fed4c5 2024-11-27T00:14:30,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c5f958023dd249f69cd55f7987fed4c5, entries=150, sequenceid=269, filesize=12.0 K 2024-11-27T00:14:30,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/ff66630c64874d8a867a8c0377b23a17 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ff66630c64874d8a867a8c0377b23a17 2024-11-27T00:14:30,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ff66630c64874d8a867a8c0377b23a17, entries=150, sequenceid=269, filesize=12.0 K 2024-11-27T00:14:30,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.62 KB/64120, heapSize ~174.28 KB/178464, currentSize=114.05 KB/116790 for e7356b094232ddb421ff69892486ac78 in 638ms, sequenceid=269, compaction requested=false 2024-11-27T00:14:30,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:30,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T00:14:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:30,432 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:14:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:30,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a0b13a42a7914366a23267fd4ddb5e87 is 50, key is test_row_0/A:col10/1732666469776/Put/seqid=0 2024-11-27T00:14:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666530451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666530454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666530453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741900_1076 (size=12301) 2024-11-27T00:14:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666530460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666530462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666530564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666530574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666530574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666530574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666530774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666530784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666530784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666530784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:30,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T00:14:30,863 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a0b13a42a7914366a23267fd4ddb5e87 2024-11-27T00:14:30,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bf4919eb60af4ea984adef719c4189de is 50, key is test_row_0/B:col10/1732666469776/Put/seqid=0 2024-11-27T00:14:30,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741901_1077 (size=12301) 2024-11-27T00:14:30,896 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bf4919eb60af4ea984adef719c4189de 2024-11-27T00:14:30,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/f867a35a622048f98d87c164dc83d415 is 50, key is test_row_0/C:col10/1732666469776/Put/seqid=0 2024-11-27T00:14:30,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741902_1078 (size=12301) 2024-11-27T00:14:30,928 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/f867a35a622048f98d87c164dc83d415 2024-11-27T00:14:30,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a0b13a42a7914366a23267fd4ddb5e87 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a0b13a42a7914366a23267fd4ddb5e87 2024-11-27T00:14:30,943 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a0b13a42a7914366a23267fd4ddb5e87, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T00:14:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bf4919eb60af4ea984adef719c4189de as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf4919eb60af4ea984adef719c4189de 2024-11-27T00:14:30,956 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf4919eb60af4ea984adef719c4189de, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T00:14:30,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/f867a35a622048f98d87c164dc83d415 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f867a35a622048f98d87c164dc83d415 2024-11-27T00:14:30,965 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f867a35a622048f98d87c164dc83d415, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T00:14:30,967 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e7356b094232ddb421ff69892486ac78 in 535ms, sequenceid=293, compaction requested=true 2024-11-27T00:14:30,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:30,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:30,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-27T00:14:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-27T00:14:30,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T00:14:30,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3090 sec 2024-11-27T00:14:30,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3220 sec 2024-11-27T00:14:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:30,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:30,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:30,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:30,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:30,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:30,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:30,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:30,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/324bc94975174d8f8e17876a92d803fd is 50, key is test_row_0/A:col10/1732666470980/Put/seqid=0 2024-11-27T00:14:31,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741903_1079 (size=14741) 2024-11-27T00:14:31,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/324bc94975174d8f8e17876a92d803fd 2024-11-27T00:14:31,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/b84eaf052b9b4185a4e311fc6b03cab6 is 50, key is test_row_0/B:col10/1732666470980/Put/seqid=0 2024-11-27T00:14:31,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741904_1080 (size=12301) 2024-11-27T00:14:31,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/b84eaf052b9b4185a4e311fc6b03cab6 2024-11-27T00:14:31,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666531076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/f9e8d5978b1349f38b281a5844d8e17b is 50, key is test_row_0/C:col10/1732666470980/Put/seqid=0 2024-11-27T00:14:31,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666531084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666531094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666531094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741905_1081 (size=12301) 2024-11-27T00:14:31,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666531104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666531183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666531394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/f9e8d5978b1349f38b281a5844d8e17b 2024-11-27T00:14:31,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/324bc94975174d8f8e17876a92d803fd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/324bc94975174d8f8e17876a92d803fd 2024-11-27T00:14:31,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/324bc94975174d8f8e17876a92d803fd, entries=200, sequenceid=307, filesize=14.4 K 2024-11-27T00:14:31,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/b84eaf052b9b4185a4e311fc6b03cab6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/b84eaf052b9b4185a4e311fc6b03cab6 2024-11-27T00:14:31,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/b84eaf052b9b4185a4e311fc6b03cab6, entries=150, sequenceid=307, filesize=12.0 K 2024-11-27T00:14:31,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/f9e8d5978b1349f38b281a5844d8e17b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f9e8d5978b1349f38b281a5844d8e17b 2024-11-27T00:14:31,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f9e8d5978b1349f38b281a5844d8e17b, entries=150, sequenceid=307, filesize=12.0 K 2024-11-27T00:14:31,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e7356b094232ddb421ff69892486ac78 in 553ms, sequenceid=307, compaction requested=true 2024-11-27T00:14:31,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:31,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:31,539 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:31,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:31,539 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:31,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:31,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:31,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:31,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:31,541 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54582 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:31,541 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:31,541 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:31,541 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:31,541 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:31,541 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:31,541 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a1e50698e74f40a3a8f2eda1fb98b7d8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b99862942db944808153c82712a8cb92, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a0b13a42a7914366a23267fd4ddb5e87, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/324bc94975174d8f8e17876a92d803fd] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=53.3 K 2024-11-27T00:14:31,541 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/38dc10e6fdb44ddc9bc6c41fa6b1155b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c5f958023dd249f69cd55f7987fed4c5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf4919eb60af4ea984adef719c4189de, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/b84eaf052b9b4185a4e311fc6b03cab6] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=48.5 K 2024-11-27T00:14:31,542 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 38dc10e6fdb44ddc9bc6c41fa6b1155b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732666469458 2024-11-27T00:14:31,542 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e50698e74f40a3a8f2eda1fb98b7d8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732666469458 2024-11-27T00:14:31,542 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f958023dd249f69cd55f7987fed4c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732666469630 2024-11-27T00:14:31,543 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99862942db944808153c82712a8cb92, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732666469630 2024-11-27T00:14:31,543 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting bf4919eb60af4ea984adef719c4189de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732666469776 2024-11-27T00:14:31,543 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0b13a42a7914366a23267fd4ddb5e87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732666469776 2024-11-27T00:14:31,543 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 324bc94975174d8f8e17876a92d803fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732666470445 2024-11-27T00:14:31,543 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b84eaf052b9b4185a4e311fc6b03cab6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732666470447 2024-11-27T00:14:31,566 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:31,568 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/a108c4986c8549c3afd32704fe7bf963 is 50, key is test_row_0/B:col10/1732666470980/Put/seqid=0 2024-11-27T00:14:31,569 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:31,570 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/eea0e03d1aac44abb414fbbda5caf994 is 50, key is test_row_0/A:col10/1732666470980/Put/seqid=0 2024-11-27T00:14:31,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741907_1083 (size=13085) 2024-11-27T00:14:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:31,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:14:31,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:31,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:31,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:31,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:31,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:31,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:31,600 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/eea0e03d1aac44abb414fbbda5caf994 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/eea0e03d1aac44abb414fbbda5caf994 2024-11-27T00:14:31,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/4766f11d102c4fbfaff3171e0fa7955f is 50, key is test_row_0/A:col10/1732666471595/Put/seqid=0 2024-11-27T00:14:31,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741906_1082 (size=13085) 2024-11-27T00:14:31,613 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into eea0e03d1aac44abb414fbbda5caf994(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:31,613 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:31,614 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=12, startTime=1732666471539; duration=0sec 2024-11-27T00:14:31,614 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:31,614 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:31,614 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:31,616 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:31,617 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:31,617 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:31,617 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/6658cfca588c4c519235eb2fd03883eb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ff66630c64874d8a867a8c0377b23a17, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f867a35a622048f98d87c164dc83d415, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f9e8d5978b1349f38b281a5844d8e17b] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=48.5 K 2024-11-27T00:14:31,618 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6658cfca588c4c519235eb2fd03883eb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732666469458 2024-11-27T00:14:31,619 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff66630c64874d8a867a8c0377b23a17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732666469630 2024-11-27T00:14:31,620 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f867a35a622048f98d87c164dc83d415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732666469776 2024-11-27T00:14:31,620 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9e8d5978b1349f38b281a5844d8e17b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732666470447 2024-11-27T00:14:31,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741908_1084 (size=14741) 2024-11-27T00:14:31,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/4766f11d102c4fbfaff3171e0fa7955f 2024-11-27T00:14:31,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666531624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666531632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666531632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666531634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/075ce6b7d36f40608f0ca14778dbbb9a is 50, key is test_row_0/B:col10/1732666471595/Put/seqid=0 2024-11-27T00:14:31,647 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:31,648 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/ce9d8a3f17814d66b5e04727dd083d52 is 50, key is test_row_0/C:col10/1732666470980/Put/seqid=0 2024-11-27T00:14:31,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741909_1085 (size=12301) 2024-11-27T00:14:31,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/075ce6b7d36f40608f0ca14778dbbb9a 2024-11-27T00:14:31,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666531704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741910_1086 (size=13085) 2024-11-27T00:14:31,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/b26e5463875040d9bea522a643939bf9 is 50, key is test_row_0/C:col10/1732666471595/Put/seqid=0 2024-11-27T00:14:31,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741911_1087 (size=12301) 2024-11-27T00:14:31,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/b26e5463875040d9bea522a643939bf9 2024-11-27T00:14:31,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/4766f11d102c4fbfaff3171e0fa7955f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4766f11d102c4fbfaff3171e0fa7955f 2024-11-27T00:14:31,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666531744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4766f11d102c4fbfaff3171e0fa7955f, entries=200, sequenceid=326, filesize=14.4 K 2024-11-27T00:14:31,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/075ce6b7d36f40608f0ca14778dbbb9a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/075ce6b7d36f40608f0ca14778dbbb9a 2024-11-27T00:14:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666531754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666531755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666531755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:31,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/075ce6b7d36f40608f0ca14778dbbb9a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-27T00:14:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/b26e5463875040d9bea522a643939bf9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/b26e5463875040d9bea522a643939bf9 2024-11-27T00:14:31,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/b26e5463875040d9bea522a643939bf9, entries=150, sequenceid=326, filesize=12.0 K 2024-11-27T00:14:31,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e7356b094232ddb421ff69892486ac78 in 180ms, sequenceid=326, compaction requested=false 2024-11-27T00:14:31,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T00:14:31,804 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:31,805 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-27T00:14:31,807 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T00:14:31,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:31,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T00:14:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:31,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:31,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:31,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:31,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:31,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:31,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/981f8f0cbd93421d832a634e46e51031 is 50, key is test_row_0/A:col10/1732666471955/Put/seqid=0 2024-11-27T00:14:31,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T00:14:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:31,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:31,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741912_1088 (size=12301) 2024-11-27T00:14:31,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/981f8f0cbd93421d832a634e46e51031 2024-11-27T00:14:31,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/9770604e5cc14cc3a527c4add37621bd is 50, key is test_row_0/B:col10/1732666471955/Put/seqid=0 2024-11-27T00:14:32,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741913_1089 (size=12301) 2024-11-27T00:14:32,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/9770604e5cc14cc3a527c4add37621bd 2024-11-27T00:14:32,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666532006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666532009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666532012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,022 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/a108c4986c8549c3afd32704fe7bf963 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a108c4986c8549c3afd32704fe7bf963 2024-11-27T00:14:32,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666532018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,031 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into a108c4986c8549c3afd32704fe7bf963(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:32,031 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:32,031 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=12, startTime=1732666471539; duration=0sec 2024-11-27T00:14:32,032 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:32,032 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:32,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/bc43e51d842340bd9d28da687d65a9a1 is 50, key is test_row_0/C:col10/1732666471955/Put/seqid=0 2024-11-27T00:14:32,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741914_1090 (size=12301) 2024-11-27T00:14:32,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/bc43e51d842340bd9d28da687d65a9a1 2024-11-27T00:14:32,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/981f8f0cbd93421d832a634e46e51031 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/981f8f0cbd93421d832a634e46e51031 2024-11-27T00:14:32,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/981f8f0cbd93421d832a634e46e51031, entries=150, sequenceid=341, filesize=12.0 K 2024-11-27T00:14:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/9770604e5cc14cc3a527c4add37621bd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9770604e5cc14cc3a527c4add37621bd 2024-11-27T00:14:32,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9770604e5cc14cc3a527c4add37621bd, entries=150, sequenceid=341, filesize=12.0 K 2024-11-27T00:14:32,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/bc43e51d842340bd9d28da687d65a9a1 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bc43e51d842340bd9d28da687d65a9a1 2024-11-27T00:14:32,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bc43e51d842340bd9d28da687d65a9a1, entries=150, sequenceid=341, filesize=12.0 K 2024-11-27T00:14:32,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e7356b094232ddb421ff69892486ac78 in 133ms, sequenceid=341, compaction requested=true 2024-11-27T00:14:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:32,090 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T00:14:32,091 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:32,095 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:32,095 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:32,095 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/eea0e03d1aac44abb414fbbda5caf994, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4766f11d102c4fbfaff3171e0fa7955f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/981f8f0cbd93421d832a634e46e51031] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=39.2 K 2024-11-27T00:14:32,096 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting eea0e03d1aac44abb414fbbda5caf994, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732666470447 2024-11-27T00:14:32,096 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4766f11d102c4fbfaff3171e0fa7955f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732666471045 2024-11-27T00:14:32,098 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 981f8f0cbd93421d832a634e46e51031, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732666471628 2024-11-27T00:14:32,106 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:32,107 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/f667b274a06d46bb978d77e4fc8965f3 is 50, key is test_row_0/A:col10/1732666471955/Put/seqid=0 2024-11-27T00:14:32,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T00:14:32,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:32,119 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:14:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T00:14:32,126 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/ce9d8a3f17814d66b5e04727dd083d52 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ce9d8a3f17814d66b5e04727dd083d52 2024-11-27T00:14:32,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741915_1091 (size=13187) 2024-11-27T00:14:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:32,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/286b4369b6854c6798118e91e5e49348 is 50, key is test_row_0/A:col10/1732666472009/Put/seqid=0 2024-11-27T00:14:32,144 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into ce9d8a3f17814d66b5e04727dd083d52(size=12.8 K), total size for store is 36.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:32,144 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:32,144 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=12, startTime=1732666471539; duration=0sec 2024-11-27T00:14:32,144 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T00:14:32,144 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:32,144 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:32,144 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:32,146 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:32,146 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:32,146 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:32,146 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a108c4986c8549c3afd32704fe7bf963, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/075ce6b7d36f40608f0ca14778dbbb9a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9770604e5cc14cc3a527c4add37621bd] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.8 K 2024-11-27T00:14:32,148 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a108c4986c8549c3afd32704fe7bf963, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732666470447 2024-11-27T00:14:32,148 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 075ce6b7d36f40608f0ca14778dbbb9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732666471045 2024-11-27T00:14:32,149 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9770604e5cc14cc3a527c4add37621bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732666471628 2024-11-27T00:14:32,170 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/f667b274a06d46bb978d77e4fc8965f3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/f667b274a06d46bb978d77e4fc8965f3 2024-11-27T00:14:32,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666532149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741916_1092 (size=12301) 2024-11-27T00:14:32,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,181 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/286b4369b6854c6798118e91e5e49348 2024-11-27T00:14:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666532170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666532170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666532171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,187 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:32,188 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3861eb5a4e87409481dc593f749cf8c2 is 50, key is test_row_0/B:col10/1732666471955/Put/seqid=0 2024-11-27T00:14:32,188 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into f667b274a06d46bb978d77e4fc8965f3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:32,188 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:32,188 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=13, startTime=1732666472090; duration=0sec 2024-11-27T00:14:32,190 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:32,190 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:32,191 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:32,192 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:32,192 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:32,192 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:32,193 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ce9d8a3f17814d66b5e04727dd083d52, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/b26e5463875040d9bea522a643939bf9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bc43e51d842340bd9d28da687d65a9a1] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.8 K 2024-11-27T00:14:32,193 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ce9d8a3f17814d66b5e04727dd083d52, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732666470447 2024-11-27T00:14:32,193 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b26e5463875040d9bea522a643939bf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732666471045 2024-11-27T00:14:32,194 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting bc43e51d842340bd9d28da687d65a9a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732666471628 2024-11-27T00:14:32,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/63ec36059dad4b66a2a526d980fb327a is 50, key is test_row_0/B:col10/1732666472009/Put/seqid=0 2024-11-27T00:14:32,206 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:32,206 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/c2a2fdf2080f42e6ba90d8681283ec12 is 50, key is test_row_0/C:col10/1732666471955/Put/seqid=0 2024-11-27T00:14:32,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741917_1093 (size=13187) 2024-11-27T00:14:32,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741918_1094 (size=12301) 2024-11-27T00:14:32,214 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/63ec36059dad4b66a2a526d980fb327a 2024-11-27T00:14:32,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741919_1095 (size=13187) 2024-11-27T00:14:32,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/1a39460579564ad1bbeb6f5202606152 is 50, key is test_row_0/C:col10/1732666472009/Put/seqid=0 2024-11-27T00:14:32,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666532224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741920_1096 (size=12301) 2024-11-27T00:14:32,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666532274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666532284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666532294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666532294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T00:14:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666532483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666532494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666532504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666532504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,627 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3861eb5a4e87409481dc593f749cf8c2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3861eb5a4e87409481dc593f749cf8c2 2024-11-27T00:14:32,635 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/c2a2fdf2080f42e6ba90d8681283ec12 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c2a2fdf2080f42e6ba90d8681283ec12 2024-11-27T00:14:32,640 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/1a39460579564ad1bbeb6f5202606152 2024-11-27T00:14:32,646 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into 3861eb5a4e87409481dc593f749cf8c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:32,647 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:32,647 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=13, startTime=1732666472090; duration=0sec 2024-11-27T00:14:32,647 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:32,647 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:32,649 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into c2a2fdf2080f42e6ba90d8681283ec12(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:32,649 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:32,649 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=13, startTime=1732666472090; duration=0sec 2024-11-27T00:14:32,650 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:32,650 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/286b4369b6854c6798118e91e5e49348 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/286b4369b6854c6798118e91e5e49348 2024-11-27T00:14:32,664 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/286b4369b6854c6798118e91e5e49348, entries=150, sequenceid=360, filesize=12.0 K 2024-11-27T00:14:32,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/63ec36059dad4b66a2a526d980fb327a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/63ec36059dad4b66a2a526d980fb327a 2024-11-27T00:14:32,699 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/63ec36059dad4b66a2a526d980fb327a, entries=150, sequenceid=360, filesize=12.0 K 2024-11-27T00:14:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/1a39460579564ad1bbeb6f5202606152 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1a39460579564ad1bbeb6f5202606152 2024-11-27T00:14:32,716 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1a39460579564ad1bbeb6f5202606152, entries=150, sequenceid=360, filesize=12.0 K 2024-11-27T00:14:32,717 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e7356b094232ddb421ff69892486ac78 in 598ms, sequenceid=360, compaction requested=false 2024-11-27T00:14:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-27T00:14:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-27T00:14:32,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T00:14:32,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 910 msec 2024-11-27T00:14:32,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 921 msec 2024-11-27T00:14:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:32,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:14:32,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:32,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:32,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:32,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:32,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:32,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:32,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/5f6484b65a7a47baaf5f702164856bee is 50, key is test_row_0/A:col10/1732666472794/Put/seqid=0 2024-11-27T00:14:32,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666532858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666532860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666532860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666532863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741921_1097 (size=14741) 2024-11-27T00:14:32,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/5f6484b65a7a47baaf5f702164856bee 2024-11-27T00:14:32,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/e69885f185854ba0bbbbc4b0878487ca is 50, key is test_row_0/B:col10/1732666472794/Put/seqid=0 2024-11-27T00:14:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T00:14:32,944 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:32,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-27T00:14:32,947 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T00:14:32,949 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:32,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:32,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741922_1098 (size=12301) 2024-11-27T00:14:32,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666532985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666532985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666532985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666532985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T00:14:33,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:33,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:33,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666533194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666533194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666533195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666533195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666533245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,252 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in e7356b094232ddb421ff69892486ac78 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:14:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T00:14:33,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:33,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:33,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/e69885f185854ba0bbbbc4b0878487ca 2024-11-27T00:14:33,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/80272b0e71b84dd19c823cf51ece7f61 is 50, key is test_row_0/C:col10/1732666472794/Put/seqid=0 2024-11-27T00:14:33,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741923_1099 (size=12301) 2024-11-27T00:14:33,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/80272b0e71b84dd19c823cf51ece7f61 2024-11-27T00:14:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/5f6484b65a7a47baaf5f702164856bee as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/5f6484b65a7a47baaf5f702164856bee 2024-11-27T00:14:33,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/5f6484b65a7a47baaf5f702164856bee, entries=200, sequenceid=379, filesize=14.4 K 2024-11-27T00:14:33,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:33,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/e69885f185854ba0bbbbc4b0878487ca as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e69885f185854ba0bbbbc4b0878487ca 2024-11-27T00:14:33,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e69885f185854ba0bbbbc4b0878487ca, entries=150, sequenceid=379, filesize=12.0 K 2024-11-27T00:14:33,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/80272b0e71b84dd19c823cf51ece7f61 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/80272b0e71b84dd19c823cf51ece7f61 2024-11-27T00:14:33,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/80272b0e71b84dd19c823cf51ece7f61, entries=150, sequenceid=379, filesize=12.0 K 2024-11-27T00:14:33,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e7356b094232ddb421ff69892486ac78 in 633ms, sequenceid=379, compaction requested=true 2024-11-27T00:14:33,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:33,446 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:33,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:33,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:33,447 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:33,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:33,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:33,449 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:33,449 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:33,449 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,449 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3861eb5a4e87409481dc593f749cf8c2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/63ec36059dad4b66a2a526d980fb327a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e69885f185854ba0bbbbc4b0878487ca] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.9 K 2024-11-27T00:14:33,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:33,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:33,450 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3861eb5a4e87409481dc593f749cf8c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732666471628 2024-11-27T00:14:33,450 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 63ec36059dad4b66a2a526d980fb327a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732666472003 2024-11-27T00:14:33,451 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:33,451 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:33,451 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,451 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/f667b274a06d46bb978d77e4fc8965f3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/286b4369b6854c6798118e91e5e49348, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/5f6484b65a7a47baaf5f702164856bee] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=39.3 K 2024-11-27T00:14:33,452 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e69885f185854ba0bbbbc4b0878487ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666472151 2024-11-27T00:14:33,457 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f667b274a06d46bb978d77e4fc8965f3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732666471628 2024-11-27T00:14:33,458 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 286b4369b6854c6798118e91e5e49348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732666472003 2024-11-27T00:14:33,458 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6484b65a7a47baaf5f702164856bee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666472149 2024-11-27T00:14:33,474 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:33,475 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ce89acc108484757ab2109f4ddce3293 is 50, key is test_row_0/B:col10/1732666472794/Put/seqid=0 2024-11-27T00:14:33,507 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:33,508 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/292c004ad0434c669aa4efd6f5f317ce is 50, key is test_row_0/A:col10/1732666472794/Put/seqid=0 2024-11-27T00:14:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:33,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:14:33,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:33,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:33,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:33,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:33,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:33,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:33,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741924_1100 (size=13289) 2024-11-27T00:14:33,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/00c43d15e19e453280ee78e6d2acc3ea is 50, key is test_row_0/A:col10/1732666472839/Put/seqid=0 2024-11-27T00:14:33,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741925_1101 (size=13289) 2024-11-27T00:14:33,542 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/292c004ad0434c669aa4efd6f5f317ce as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/292c004ad0434c669aa4efd6f5f317ce 2024-11-27T00:14:33,554 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into 292c004ad0434c669aa4efd6f5f317ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:33,554 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:33,554 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=13, startTime=1732666473446; duration=0sec 2024-11-27T00:14:33,559 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:33,559 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:33,559 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:33,560 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:33,561 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:33,561 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,561 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c2a2fdf2080f42e6ba90d8681283ec12, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1a39460579564ad1bbeb6f5202606152, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/80272b0e71b84dd19c823cf51ece7f61] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=36.9 K 2024-11-27T00:14:33,561 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a2fdf2080f42e6ba90d8681283ec12, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732666471628 2024-11-27T00:14:33,562 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a39460579564ad1bbeb6f5202606152, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732666472003 2024-11-27T00:14:33,562 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80272b0e71b84dd19c823cf51ece7f61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666472151 2024-11-27T00:14:33,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741926_1102 (size=12301) 2024-11-27T00:14:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T00:14:33,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666533555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666533555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:33,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,582 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:33,583 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/c8699b52dd99418290e617447d1ebe27 is 50, key is test_row_0/C:col10/1732666472794/Put/seqid=0 2024-11-27T00:14:33,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666533557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666533556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741927_1103 (size=13289) 2024-11-27T00:14:33,646 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/c8699b52dd99418290e617447d1ebe27 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c8699b52dd99418290e617447d1ebe27 2024-11-27T00:14:33,654 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into c8699b52dd99418290e617447d1ebe27(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:33,654 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:33,654 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=13, startTime=1732666473449; duration=0sec 2024-11-27T00:14:33,655 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:33,655 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:33,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666533684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666533684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666533704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666533704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:33,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:33,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:33,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:33,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666533894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666533895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666533914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666533924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:33,933 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ce89acc108484757ab2109f4ddce3293 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ce89acc108484757ab2109f4ddce3293 2024-11-27T00:14:33,946 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into ce89acc108484757ab2109f4ddce3293(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:33,946 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:33,946 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=13, startTime=1732666473447; duration=0sec 2024-11-27T00:14:33,946 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:33,946 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:33,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/00c43d15e19e453280ee78e6d2acc3ea 2024-11-27T00:14:34,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ea8b3367dc9043c3a0ff73f0f2338581 is 50, key is test_row_0/B:col10/1732666472839/Put/seqid=0 2024-11-27T00:14:34,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741928_1104 (size=12301) 2024-11-27T00:14:34,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:34,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T00:14:34,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:34,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666534204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666534204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666534224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666534234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:34,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:34,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:34,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ea8b3367dc9043c3a0ff73f0f2338581 2024-11-27T00:14:34,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/9dd28bf7d827468fb7dd99a74a8ead58 is 50, key is test_row_0/C:col10/1732666472839/Put/seqid=0 2024-11-27T00:14:34,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741929_1105 (size=12301) 2024-11-27T00:14:34,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/9dd28bf7d827468fb7dd99a74a8ead58 2024-11-27T00:14:34,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/00c43d15e19e453280ee78e6d2acc3ea as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/00c43d15e19e453280ee78e6d2acc3ea 2024-11-27T00:14:34,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/00c43d15e19e453280ee78e6d2acc3ea, entries=150, sequenceid=398, filesize=12.0 K 2024-11-27T00:14:34,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/ea8b3367dc9043c3a0ff73f0f2338581 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ea8b3367dc9043c3a0ff73f0f2338581 2024-11-27T00:14:34,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ea8b3367dc9043c3a0ff73f0f2338581, entries=150, sequenceid=398, filesize=12.0 K 2024-11-27T00:14:34,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/9dd28bf7d827468fb7dd99a74a8ead58 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/9dd28bf7d827468fb7dd99a74a8ead58 2024-11-27T00:14:34,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/9dd28bf7d827468fb7dd99a74a8ead58, entries=150, sequenceid=398, filesize=12.0 K 2024-11-27T00:14:34,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e7356b094232ddb421ff69892486ac78 in 994ms, sequenceid=398, compaction requested=false 2024-11-27T00:14:34,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:34,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T00:14:34,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:34,511 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:34,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a05550d5e606448f9daf743bf2f4710f is 50, key is test_row_0/A:col10/1732666473554/Put/seqid=0 2024-11-27T00:14:34,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741930_1106 (size=12301) 2024-11-27T00:14:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:34,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:34,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666534747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666534749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666534750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666534757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666534864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666534864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666534865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666534865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:34,954 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a05550d5e606448f9daf743bf2f4710f 2024-11-27T00:14:34,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/493c1a76fee64e98835862bd17a83823 is 50, key is test_row_0/B:col10/1732666473554/Put/seqid=0 2024-11-27T00:14:34,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741931_1107 (size=12301) 2024-11-27T00:14:35,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666535074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666535074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666535074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666535074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T00:14:35,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666535274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,276 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in e7356b094232ddb421ff69892486ac78 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:14:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,367 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/493c1a76fee64e98835862bd17a83823 2024-11-27T00:14:35,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/78cf4779902b44cc80ac8a66086f35b7 is 50, key is test_row_0/C:col10/1732666473554/Put/seqid=0 2024-11-27T00:14:35,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741932_1108 (size=12301) 2024-11-27T00:14:35,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666535383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666535384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666535384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666535384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,781 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/78cf4779902b44cc80ac8a66086f35b7 2024-11-27T00:14:35,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a05550d5e606448f9daf743bf2f4710f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a05550d5e606448f9daf743bf2f4710f 2024-11-27T00:14:35,793 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a05550d5e606448f9daf743bf2f4710f, entries=150, sequenceid=415, filesize=12.0 K 2024-11-27T00:14:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/493c1a76fee64e98835862bd17a83823 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/493c1a76fee64e98835862bd17a83823 2024-11-27T00:14:35,800 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/493c1a76fee64e98835862bd17a83823, entries=150, sequenceid=415, filesize=12.0 K 2024-11-27T00:14:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/78cf4779902b44cc80ac8a66086f35b7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/78cf4779902b44cc80ac8a66086f35b7 2024-11-27T00:14:35,810 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/78cf4779902b44cc80ac8a66086f35b7, entries=150, sequenceid=415, filesize=12.0 K 2024-11-27T00:14:35,811 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e7356b094232ddb421ff69892486ac78 in 1301ms, sequenceid=415, compaction requested=true 2024-11-27T00:14:35,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:35,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:35,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-27T00:14:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-27T00:14:35,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-27T00:14:35,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8650 sec 2024-11-27T00:14:35,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.8720 sec 2024-11-27T00:14:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:35,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:14:35,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:35,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:35,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:35,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:35,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:35,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:35,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0ab9d98434ee46d39d32b00f64bad53a is 50, key is test_row_0/A:col10/1732666475895/Put/seqid=0 2024-11-27T00:14:35,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741933_1109 (size=14741) 2024-11-27T00:14:35,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0ab9d98434ee46d39d32b00f64bad53a 2024-11-27T00:14:35,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666535919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666535920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666535923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/7ed26768aa7b4f57a43cb8e8e610ccc8 is 50, key is test_row_0/B:col10/1732666475895/Put/seqid=0 2024-11-27T00:14:35,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666535923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:35,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741934_1110 (size=12301) 2024-11-27T00:14:36,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666536034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666536034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666536034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666536044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666536244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666536244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666536244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666536253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/7ed26768aa7b4f57a43cb8e8e610ccc8 2024-11-27T00:14:36,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/a8d88e56ed944d87981dd65d57bdf796 is 50, key is test_row_0/C:col10/1732666475895/Put/seqid=0 2024-11-27T00:14:36,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741935_1111 (size=12301) 2024-11-27T00:14:36,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/a8d88e56ed944d87981dd65d57bdf796 2024-11-27T00:14:36,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0ab9d98434ee46d39d32b00f64bad53a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0ab9d98434ee46d39d32b00f64bad53a 2024-11-27T00:14:36,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0ab9d98434ee46d39d32b00f64bad53a, entries=200, sequenceid=435, filesize=14.4 K 2024-11-27T00:14:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/7ed26768aa7b4f57a43cb8e8e610ccc8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/7ed26768aa7b4f57a43cb8e8e610ccc8 2024-11-27T00:14:36,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/7ed26768aa7b4f57a43cb8e8e610ccc8, entries=150, sequenceid=435, filesize=12.0 K 2024-11-27T00:14:36,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/a8d88e56ed944d87981dd65d57bdf796 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/a8d88e56ed944d87981dd65d57bdf796 2024-11-27T00:14:36,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/a8d88e56ed944d87981dd65d57bdf796, entries=150, sequenceid=435, filesize=12.0 K 2024-11-27T00:14:36,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e7356b094232ddb421ff69892486ac78 in 508ms, sequenceid=435, compaction requested=true 2024-11-27T00:14:36,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:36,406 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:36,407 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:36,407 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:36,407 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:36,407 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/292c004ad0434c669aa4efd6f5f317ce, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/00c43d15e19e453280ee78e6d2acc3ea, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a05550d5e606448f9daf743bf2f4710f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0ab9d98434ee46d39d32b00f64bad53a] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=51.4 K 2024-11-27T00:14:36,408 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 292c004ad0434c669aa4efd6f5f317ce, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666472151 2024-11-27T00:14:36,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:36,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:36,408 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:36,408 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00c43d15e19e453280ee78e6d2acc3ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732666472839 2024-11-27T00:14:36,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:36,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:36,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:36,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:36,409 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a05550d5e606448f9daf743bf2f4710f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732666473527 2024-11-27T00:14:36,409 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ab9d98434ee46d39d32b00f64bad53a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732666474740 2024-11-27T00:14:36,410 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:36,410 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:36,410 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:36,410 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ce89acc108484757ab2109f4ddce3293, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ea8b3367dc9043c3a0ff73f0f2338581, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/493c1a76fee64e98835862bd17a83823, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/7ed26768aa7b4f57a43cb8e8e610ccc8] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=49.0 K 2024-11-27T00:14:36,411 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ce89acc108484757ab2109f4ddce3293, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666472151 2024-11-27T00:14:36,412 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8b3367dc9043c3a0ff73f0f2338581, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732666472839 2024-11-27T00:14:36,412 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 493c1a76fee64e98835862bd17a83823, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732666473527 2024-11-27T00:14:36,413 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed26768aa7b4f57a43cb8e8e610ccc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732666474740 2024-11-27T00:14:36,431 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:36,432 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:36,432 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/60b40d915e7a4d73a287c7d0f4e08525 is 50, key is test_row_0/B:col10/1732666475895/Put/seqid=0 2024-11-27T00:14:36,434 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/1dad3da8218240c38bc8a297fe1f0314 is 50, key is test_row_0/A:col10/1732666475895/Put/seqid=0 2024-11-27T00:14:36,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741937_1113 (size=13425) 2024-11-27T00:14:36,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741936_1112 (size=13425) 2024-11-27T00:14:36,459 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/60b40d915e7a4d73a287c7d0f4e08525 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/60b40d915e7a4d73a287c7d0f4e08525 2024-11-27T00:14:36,469 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into 60b40d915e7a4d73a287c7d0f4e08525(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:36,469 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:36,469 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=12, startTime=1732666476408; duration=0sec 2024-11-27T00:14:36,469 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:36,469 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:36,469 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:36,471 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:36,471 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:36,471 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:36,472 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c8699b52dd99418290e617447d1ebe27, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/9dd28bf7d827468fb7dd99a74a8ead58, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/78cf4779902b44cc80ac8a66086f35b7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/a8d88e56ed944d87981dd65d57bdf796] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=49.0 K 2024-11-27T00:14:36,473 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c8699b52dd99418290e617447d1ebe27, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666472151 2024-11-27T00:14:36,474 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dd28bf7d827468fb7dd99a74a8ead58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732666472839 2024-11-27T00:14:36,474 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 78cf4779902b44cc80ac8a66086f35b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732666473527 2024-11-27T00:14:36,475 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting a8d88e56ed944d87981dd65d57bdf796, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732666474740 2024-11-27T00:14:36,497 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:36,498 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/bf54a3b5220647ba8ed90f858ecb0190 is 50, key is test_row_0/C:col10/1732666475895/Put/seqid=0 2024-11-27T00:14:36,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741938_1114 (size=13425) 2024-11-27T00:14:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:36,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:36,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:36,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:36,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:36,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:36,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:36,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:36,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/79e64a27ea4f49d6bb85bbeaea6e0fe9 is 50, key is test_row_0/A:col10/1732666475919/Put/seqid=0 2024-11-27T00:14:36,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741939_1115 (size=14741) 2024-11-27T00:14:36,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/79e64a27ea4f49d6bb85bbeaea6e0fe9 2024-11-27T00:14:36,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/5075abc83a5e426c9745aeff3261cd04 is 50, key is test_row_0/B:col10/1732666475919/Put/seqid=0 2024-11-27T00:14:36,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741940_1116 (size=12301) 2024-11-27T00:14:36,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666536579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666536579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666536581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666536585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666536694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666536694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666536694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666536694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,859 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/1dad3da8218240c38bc8a297fe1f0314 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/1dad3da8218240c38bc8a297fe1f0314 2024-11-27T00:14:36,866 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into 1dad3da8218240c38bc8a297fe1f0314(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:36,866 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:36,866 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=12, startTime=1732666476406; duration=0sec 2024-11-27T00:14:36,866 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:36,866 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:36,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666536904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666536904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666536904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666536904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:36,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:36,908 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/bf54a3b5220647ba8ed90f858ecb0190 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bf54a3b5220647ba8ed90f858ecb0190 2024-11-27T00:14:36,913 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into bf54a3b5220647ba8ed90f858ecb0190(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:36,913 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:36,913 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=12, startTime=1732666476409; duration=0sec 2024-11-27T00:14:36,913 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:36,913 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:36,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/5075abc83a5e426c9745aeff3261cd04 2024-11-27T00:14:36,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/bee3cb4d103a4d05b4d75bc90d38165e is 50, key is test_row_0/C:col10/1732666475919/Put/seqid=0 2024-11-27T00:14:37,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741941_1117 (size=12301) 2024-11-27T00:14:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T00:14:37,114 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:37,115 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-27T00:14:37,118 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T00:14:37,119 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:37,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:37,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666537213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666537214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666537214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666537214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T00:14:37,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T00:14:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:37,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:37,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/bee3cb4d103a4d05b4d75bc90d38165e 2024-11-27T00:14:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/79e64a27ea4f49d6bb85bbeaea6e0fe9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/79e64a27ea4f49d6bb85bbeaea6e0fe9 2024-11-27T00:14:37,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/79e64a27ea4f49d6bb85bbeaea6e0fe9, entries=200, sequenceid=450, filesize=14.4 K 2024-11-27T00:14:37,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/5075abc83a5e426c9745aeff3261cd04 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/5075abc83a5e426c9745aeff3261cd04 2024-11-27T00:14:37,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T00:14:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:37,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:37,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:37,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:37,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/5075abc83a5e426c9745aeff3261cd04, entries=150, sequenceid=450, filesize=12.0 K 2024-11-27T00:14:37,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/bee3cb4d103a4d05b4d75bc90d38165e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bee3cb4d103a4d05b4d75bc90d38165e 2024-11-27T00:14:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T00:14:37,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bee3cb4d103a4d05b4d75bc90d38165e, entries=150, sequenceid=450, filesize=12.0 K 2024-11-27T00:14:37,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e7356b094232ddb421ff69892486ac78 in 885ms, sequenceid=450, compaction requested=false 2024-11-27T00:14:37,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:37,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T00:14:37,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:37,579 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:14:37,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:37,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:37,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:37,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:37,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:37,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:37,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/28df23f674604222a44dd3c1ebd92f26 is 50, key is test_row_0/A:col10/1732666476568/Put/seqid=0 2024-11-27T00:14:37,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741942_1118 (size=12301) 2024-11-27T00:14:37,594 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/28df23f674604222a44dd3c1ebd92f26 2024-11-27T00:14:37,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3e953ff37c5e463ea32ab93dc880be15 is 50, key is test_row_0/B:col10/1732666476568/Put/seqid=0 2024-11-27T00:14:37,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741943_1119 (size=12301) 2024-11-27T00:14:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:37,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T00:14:37,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666537734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666537736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666537743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666537735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666537854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666537854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666537854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666537855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,011 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3e953ff37c5e463ea32ab93dc880be15 2024-11-27T00:14:38,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/971c6d3afd3f4146bc37adde41c2912a is 50, key is test_row_0/C:col10/1732666476568/Put/seqid=0 2024-11-27T00:14:38,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741944_1120 (size=12301) 2024-11-27T00:14:38,035 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/971c6d3afd3f4146bc37adde41c2912a 2024-11-27T00:14:38,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/28df23f674604222a44dd3c1ebd92f26 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/28df23f674604222a44dd3c1ebd92f26 2024-11-27T00:14:38,052 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/28df23f674604222a44dd3c1ebd92f26, entries=150, sequenceid=470, filesize=12.0 K 2024-11-27T00:14:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3e953ff37c5e463ea32ab93dc880be15 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e953ff37c5e463ea32ab93dc880be15 2024-11-27T00:14:38,065 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e953ff37c5e463ea32ab93dc880be15, entries=150, sequenceid=470, filesize=12.0 K 2024-11-27T00:14:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666538064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666538064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666538064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666538064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/971c6d3afd3f4146bc37adde41c2912a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/971c6d3afd3f4146bc37adde41c2912a 2024-11-27T00:14:38,078 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/971c6d3afd3f4146bc37adde41c2912a, entries=150, sequenceid=470, filesize=12.0 K 2024-11-27T00:14:38,079 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e7356b094232ddb421ff69892486ac78 in 500ms, sequenceid=470, compaction requested=true 2024-11-27T00:14:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-27T00:14:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-27T00:14:38,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T00:14:38,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 962 msec 2024-11-27T00:14:38,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 969 msec 2024-11-27T00:14:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T00:14:38,255 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:38,256 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-27T00:14:38,259 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T00:14:38,259 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:38,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T00:14:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:38,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:14:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:38,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/03c79867dadb4eb5b6401d734cdfd882 is 50, key is test_row_0/A:col10/1732666477727/Put/seqid=0 2024-11-27T00:14:38,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666538400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666538402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666538407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666538408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T00:14:38,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:38,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:38,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:38,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:38,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:38,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741945_1121 (size=17181) 2024-11-27T00:14:38,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/03c79867dadb4eb5b6401d734cdfd882 2024-11-27T00:14:38,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bba56710af954d5498c598e398b9cf7b is 50, key is test_row_0/B:col10/1732666477727/Put/seqid=0 2024-11-27T00:14:38,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741946_1122 (size=12301) 2024-11-27T00:14:38,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bba56710af954d5498c598e398b9cf7b 2024-11-27T00:14:38,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/482391a485db44f6b718f033b2db7975 is 50, key is test_row_0/C:col10/1732666477727/Put/seqid=0 2024-11-27T00:14:38,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741947_1123 (size=12301) 2024-11-27T00:14:38,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/482391a485db44f6b718f033b2db7975 2024-11-27T00:14:38,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/03c79867dadb4eb5b6401d734cdfd882 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/03c79867dadb4eb5b6401d734cdfd882 2024-11-27T00:14:38,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/03c79867dadb4eb5b6401d734cdfd882, entries=250, sequenceid=486, filesize=16.8 K 2024-11-27T00:14:38,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bba56710af954d5498c598e398b9cf7b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bba56710af954d5498c598e398b9cf7b 2024-11-27T00:14:38,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666538514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666538514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666538524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666538524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bba56710af954d5498c598e398b9cf7b, entries=150, sequenceid=486, filesize=12.0 K 2024-11-27T00:14:38,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/482391a485db44f6b718f033b2db7975 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/482391a485db44f6b718f033b2db7975 2024-11-27T00:14:38,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/482391a485db44f6b718f033b2db7975, entries=150, sequenceid=486, filesize=12.0 K 2024-11-27T00:14:38,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e7356b094232ddb421ff69892486ac78 in 166ms, sequenceid=486, compaction requested=true 2024-11-27T00:14:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:38,543 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:38,543 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:38,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:38,545 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:38,545 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:38,545 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:38,545 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:38,545 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:38,546 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/60b40d915e7a4d73a287c7d0f4e08525, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/5075abc83a5e426c9745aeff3261cd04, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e953ff37c5e463ea32ab93dc880be15, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bba56710af954d5498c598e398b9cf7b] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=49.1 K 2024-11-27T00:14:38,546 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:38,546 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/1dad3da8218240c38bc8a297fe1f0314, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/79e64a27ea4f49d6bb85bbeaea6e0fe9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/28df23f674604222a44dd3c1ebd92f26, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/03c79867dadb4eb5b6401d734cdfd882] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=56.3 K 2024-11-27T00:14:38,556 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dad3da8218240c38bc8a297fe1f0314, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732666474740 2024-11-27T00:14:38,556 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b40d915e7a4d73a287c7d0f4e08525, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732666474740 2024-11-27T00:14:38,557 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79e64a27ea4f49d6bb85bbeaea6e0fe9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732666475918 2024-11-27T00:14:38,557 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5075abc83a5e426c9745aeff3261cd04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732666475918 2024-11-27T00:14:38,558 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e953ff37c5e463ea32ab93dc880be15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732666476568 2024-11-27T00:14:38,558 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting bba56710af954d5498c598e398b9cf7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732666477727 2024-11-27T00:14:38,560 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28df23f674604222a44dd3c1ebd92f26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732666476568 2024-11-27T00:14:38,561 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c79867dadb4eb5b6401d734cdfd882, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732666477727 2024-11-27T00:14:38,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T00:14:38,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:38,571 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:14:38,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:38,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:38,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:38,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:38,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:38,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T00:14:38,580 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:38,581 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a2b68b4f59444b639d7b68e0204481f1 is 50, key is test_row_0/A:col10/1732666477727/Put/seqid=0 2024-11-27T00:14:38,587 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#112 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:38,589 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/4c76c8dd7fc7418e8b12ed7933f7fa95 is 50, key is test_row_0/B:col10/1732666477727/Put/seqid=0 2024-11-27T00:14:38,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/fdc42b6beefd419f98779ac225d9922f is 50, key is test_row_0/A:col10/1732666478405/Put/seqid=0 2024-11-27T00:14:38,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741948_1124 (size=13561) 2024-11-27T00:14:38,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741949_1125 (size=13561) 2024-11-27T00:14:38,647 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/4c76c8dd7fc7418e8b12ed7933f7fa95 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/4c76c8dd7fc7418e8b12ed7933f7fa95 2024-11-27T00:14:38,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741950_1126 (size=12301) 2024-11-27T00:14:38,654 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into 4c76c8dd7fc7418e8b12ed7933f7fa95(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:38,654 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:38,654 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=12, startTime=1732666478543; duration=0sec 2024-11-27T00:14:38,654 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:38,654 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:38,654 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:38,655 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:38,656 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:38,657 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:38,657 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bf54a3b5220647ba8ed90f858ecb0190, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bee3cb4d103a4d05b4d75bc90d38165e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/971c6d3afd3f4146bc37adde41c2912a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/482391a485db44f6b718f033b2db7975] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=49.1 K 2024-11-27T00:14:38,658 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting bf54a3b5220647ba8ed90f858ecb0190, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1732666474740 2024-11-27T00:14:38,659 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting bee3cb4d103a4d05b4d75bc90d38165e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732666475918 2024-11-27T00:14:38,659 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 971c6d3afd3f4146bc37adde41c2912a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732666476568 2024-11-27T00:14:38,663 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 482391a485db44f6b718f033b2db7975, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732666477727 2024-11-27T00:14:38,683 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#114 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:38,684 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/49273a2d2c534c9cb082f7bc069f0ef8 is 50, key is test_row_0/C:col10/1732666477727/Put/seqid=0 2024-11-27T00:14:38,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741951_1127 (size=13561) 2024-11-27T00:14:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:38,725 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/49273a2d2c534c9cb082f7bc069f0ef8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/49273a2d2c534c9cb082f7bc069f0ef8 2024-11-27T00:14:38,731 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into 49273a2d2c534c9cb082f7bc069f0ef8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:38,731 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:38,731 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=12, startTime=1732666478543; duration=0sec 2024-11-27T00:14:38,731 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:38,731 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:38,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666538746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666538748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666538751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666538757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666538864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666538864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666538864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666538864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T00:14:39,038 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a2b68b4f59444b639d7b68e0204481f1 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a2b68b4f59444b639d7b68e0204481f1 2024-11-27T00:14:39,046 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into a2b68b4f59444b639d7b68e0204481f1(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:39,046 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:39,047 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=12, startTime=1732666478543; duration=0sec 2024-11-27T00:14:39,047 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:39,047 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:39,055 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/fdc42b6beefd419f98779ac225d9922f 2024-11-27T00:14:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bf0033714fbb4b2995b5344ad5b22bb9 is 50, key is test_row_0/B:col10/1732666478405/Put/seqid=0 2024-11-27T00:14:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666539074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666539074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666539074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666539079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741952_1128 (size=12301) 2024-11-27T00:14:39,102 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bf0033714fbb4b2995b5344ad5b22bb9 2024-11-27T00:14:39,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/3e3759acfb6d4a3aa9abd30bb633d1d3 is 50, key is test_row_0/C:col10/1732666478405/Put/seqid=0 2024-11-27T00:14:39,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741953_1129 (size=12301) 2024-11-27T00:14:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38932 deadline: 1732666539314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,317 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in e7356b094232ddb421ff69892486ac78 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666539384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666539384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666539385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666539385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T00:14:39,532 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/3e3759acfb6d4a3aa9abd30bb633d1d3 2024-11-27T00:14:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/fdc42b6beefd419f98779ac225d9922f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fdc42b6beefd419f98779ac225d9922f 2024-11-27T00:14:39,550 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fdc42b6beefd419f98779ac225d9922f, entries=150, sequenceid=502, filesize=12.0 K 2024-11-27T00:14:39,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/bf0033714fbb4b2995b5344ad5b22bb9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf0033714fbb4b2995b5344ad5b22bb9 2024-11-27T00:14:39,559 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf0033714fbb4b2995b5344ad5b22bb9, entries=150, sequenceid=502, filesize=12.0 K 2024-11-27T00:14:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/3e3759acfb6d4a3aa9abd30bb633d1d3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/3e3759acfb6d4a3aa9abd30bb633d1d3 2024-11-27T00:14:39,569 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/3e3759acfb6d4a3aa9abd30bb633d1d3, entries=150, sequenceid=502, filesize=12.0 K 2024-11-27T00:14:39,570 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e7356b094232ddb421ff69892486ac78 in 1000ms, sequenceid=502, compaction requested=false 2024-11-27T00:14:39,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:39,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:39,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-27T00:14:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-27T00:14:39,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T00:14:39,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3120 sec 2024-11-27T00:14:39,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.3210 sec 2024-11-27T00:14:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:39,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:14:39,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:39,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:39,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:39,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:39,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:39,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:39,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0940f71f478547bab4722e8e11559eb2 is 50, key is test_row_0/A:col10/1732666478737/Put/seqid=0 2024-11-27T00:14:39,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741954_1130 (size=12301) 2024-11-27T00:14:39,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0940f71f478547bab4722e8e11559eb2 2024-11-27T00:14:39,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666539910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666539911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666539912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666539913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:39,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/1722f598210b496796c48bc6f1c22f3f is 50, key is test_row_0/B:col10/1732666478737/Put/seqid=0 2024-11-27T00:14:39,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741955_1131 (size=12301) 2024-11-27T00:14:39,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/1722f598210b496796c48bc6f1c22f3f 2024-11-27T00:14:39,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/91708e578723421a848d13a3b8db47af is 50, key is test_row_0/C:col10/1732666478737/Put/seqid=0 2024-11-27T00:14:39,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741956_1132 (size=12301) 2024-11-27T00:14:39,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/91708e578723421a848d13a3b8db47af 2024-11-27T00:14:39,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/0940f71f478547bab4722e8e11559eb2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0940f71f478547bab4722e8e11559eb2 2024-11-27T00:14:39,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0940f71f478547bab4722e8e11559eb2, entries=150, sequenceid=522, filesize=12.0 K 2024-11-27T00:14:39,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/1722f598210b496796c48bc6f1c22f3f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1722f598210b496796c48bc6f1c22f3f 2024-11-27T00:14:39,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1722f598210b496796c48bc6f1c22f3f, entries=150, sequenceid=522, filesize=12.0 K 2024-11-27T00:14:39,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/91708e578723421a848d13a3b8db47af as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/91708e578723421a848d13a3b8db47af 2024-11-27T00:14:39,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/91708e578723421a848d13a3b8db47af, entries=150, sequenceid=522, filesize=12.0 K 2024-11-27T00:14:39,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e7356b094232ddb421ff69892486ac78 in 62ms, sequenceid=522, compaction requested=true 2024-11-27T00:14:39,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:39,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:39,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:39,958 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:39,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:39,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:39,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7356b094232ddb421ff69892486ac78:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:39,958 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:39,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:39,959 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:39,959 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:39,959 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/B is initiating minor compaction (all files) 2024-11-27T00:14:39,959 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/A is initiating minor compaction (all files) 2024-11-27T00:14:39,959 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/B in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:39,959 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/A in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:39,959 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a2b68b4f59444b639d7b68e0204481f1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fdc42b6beefd419f98779ac225d9922f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0940f71f478547bab4722e8e11559eb2] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=37.3 K 2024-11-27T00:14:39,959 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/4c76c8dd7fc7418e8b12ed7933f7fa95, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf0033714fbb4b2995b5344ad5b22bb9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1722f598210b496796c48bc6f1c22f3f] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=37.3 K 2024-11-27T00:14:39,959 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c76c8dd7fc7418e8b12ed7933f7fa95, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732666477727 2024-11-27T00:14:39,959 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2b68b4f59444b639d7b68e0204481f1, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732666477727 2024-11-27T00:14:39,960 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting bf0033714fbb4b2995b5344ad5b22bb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732666478393 2024-11-27T00:14:39,960 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc42b6beefd419f98779ac225d9922f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732666478393 2024-11-27T00:14:39,960 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1722f598210b496796c48bc6f1c22f3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732666478737 2024-11-27T00:14:39,960 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0940f71f478547bab4722e8e11559eb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732666478737 2024-11-27T00:14:39,970 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#A#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:39,970 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/d224d6ab4ea54987b2e5c1656933c470 is 50, key is test_row_0/A:col10/1732666478737/Put/seqid=0 2024-11-27T00:14:39,973 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:39,974 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/9d3335c6eadb485cb8ba10a1eb904a97 is 50, key is test_row_0/B:col10/1732666478737/Put/seqid=0 2024-11-27T00:14:39,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741957_1133 (size=13663) 2024-11-27T00:14:39,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741958_1134 (size=13663) 2024-11-27T00:14:39,990 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/9d3335c6eadb485cb8ba10a1eb904a97 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9d3335c6eadb485cb8ba10a1eb904a97 2024-11-27T00:14:39,996 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/B of e7356b094232ddb421ff69892486ac78 into 9d3335c6eadb485cb8ba10a1eb904a97(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:39,996 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:39,996 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/B, priority=13, startTime=1732666479958; duration=0sec 2024-11-27T00:14:39,996 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:39,996 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:B 2024-11-27T00:14:39,997 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:14:39,998 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:14:39,998 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e7356b094232ddb421ff69892486ac78/C is initiating minor compaction (all files) 2024-11-27T00:14:39,998 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7356b094232ddb421ff69892486ac78/C in TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:39,998 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/49273a2d2c534c9cb082f7bc069f0ef8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/3e3759acfb6d4a3aa9abd30bb633d1d3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/91708e578723421a848d13a3b8db47af] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp, totalSize=37.3 K 2024-11-27T00:14:40,000 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 49273a2d2c534c9cb082f7bc069f0ef8, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732666477727 2024-11-27T00:14:40,001 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e3759acfb6d4a3aa9abd30bb633d1d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732666478393 2024-11-27T00:14:40,002 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 91708e578723421a848d13a3b8db47af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732666478737 2024-11-27T00:14:40,016 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7356b094232ddb421ff69892486ac78#C#compaction#122 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:40,016 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/156adc7af9f44de7adb128c8099a68ec is 50, key is test_row_0/C:col10/1732666478737/Put/seqid=0 2024-11-27T00:14:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741959_1135 (size=13663) 2024-11-27T00:14:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:40,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:14:40,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:40,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:40,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:40,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:40,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:40,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:40,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/c880037ebd204468b01e358c1212514d is 50, key is test_row_0/A:col10/1732666480026/Put/seqid=0 2024-11-27T00:14:40,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741960_1136 (size=12301) 2024-11-27T00:14:40,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666540043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666540043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666540044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666540044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666540154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666540154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666540154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666540154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666540364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666540364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666540364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666540364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,382 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/d224d6ab4ea54987b2e5c1656933c470 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d224d6ab4ea54987b2e5c1656933c470 2024-11-27T00:14:40,387 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/A of e7356b094232ddb421ff69892486ac78 into d224d6ab4ea54987b2e5c1656933c470(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:40,387 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:40,387 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/A, priority=13, startTime=1732666479957; duration=0sec 2024-11-27T00:14:40,387 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:40,387 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:A 2024-11-27T00:14:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T00:14:40,414 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:40,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-27T00:14:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T00:14:40,416 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:40,417 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:40,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:40,432 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/156adc7af9f44de7adb128c8099a68ec as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/156adc7af9f44de7adb128c8099a68ec 2024-11-27T00:14:40,437 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7356b094232ddb421ff69892486ac78/C of e7356b094232ddb421ff69892486ac78 into 156adc7af9f44de7adb128c8099a68ec(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:40,437 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:40,437 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., storeName=e7356b094232ddb421ff69892486ac78/C, priority=13, startTime=1732666479958; duration=0sec 2024-11-27T00:14:40,437 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:40,437 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7356b094232ddb421ff69892486ac78:C 2024-11-27T00:14:40,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/c880037ebd204468b01e358c1212514d 2024-11-27T00:14:40,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/698a4d867ae94b5fb36b3d4c88b78177 is 50, key is test_row_0/B:col10/1732666480026/Put/seqid=0 2024-11-27T00:14:40,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741961_1137 (size=12301) 2024-11-27T00:14:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T00:14:40,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-27T00:14:40,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:40,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:40,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:40,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38906 deadline: 1732666540674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38952 deadline: 1732666540674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38912 deadline: 1732666540684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38946 deadline: 1732666540684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:14:40,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-27T00:14:40,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:40,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:40,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:40,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T00:14:40,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/698a4d867ae94b5fb36b3d4c88b78177 2024-11-27T00:14:40,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/423cc8b8ac714585b728410e814c75a6 is 50, key is test_row_0/C:col10/1732666480026/Put/seqid=0 2024-11-27T00:14:40,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741962_1138 (size=12301) 2024-11-27T00:14:40,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/423cc8b8ac714585b728410e814c75a6 2024-11-27T00:14:40,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-27T00:14:40,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:40,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:40,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/c880037ebd204468b01e358c1212514d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c880037ebd204468b01e358c1212514d 2024-11-27T00:14:40,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c880037ebd204468b01e358c1212514d, entries=150, sequenceid=541, filesize=12.0 K 2024-11-27T00:14:40,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/698a4d867ae94b5fb36b3d4c88b78177 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/698a4d867ae94b5fb36b3d4c88b78177 2024-11-27T00:14:40,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/698a4d867ae94b5fb36b3d4c88b78177, entries=150, sequenceid=541, filesize=12.0 K 2024-11-27T00:14:40,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/423cc8b8ac714585b728410e814c75a6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/423cc8b8ac714585b728410e814c75a6 2024-11-27T00:14:40,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/423cc8b8ac714585b728410e814c75a6, entries=150, sequenceid=541, filesize=12.0 K 2024-11-27T00:14:40,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e7356b094232ddb421ff69892486ac78 in 877ms, sequenceid=541, compaction requested=false 2024-11-27T00:14:40,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:40,921 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-27T00:14:40,921 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:40,922 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-27T00:14:40,922 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:40,923 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-27T00:14:40,923 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:40,926 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:40,926 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:40,926 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:40,927 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:40,927 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:40,927 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:40,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:40,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:40,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:40,928 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-27T00:14:40,929 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:40,929 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:40,929 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:40,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:41,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-27T00:14:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:41,032 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:14:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:41,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a3594c8b6f42452c8deb0ce0b39dfd1a is 50, key is test_row_0/A:col10/1732666480043/Put/seqid=0 2024-11-27T00:14:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T00:14:41,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741963_1139 (size=12301) 2024-11-27T00:14:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:41,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. as already flushing 2024-11-27T00:14:41,190 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-27T00:14:41,190 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-27T00:14:41,190 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:41,190 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:41,190 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:41,190 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:41,190 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:41,191 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:41,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:41,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:41,196 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-27T00:14:41,196 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-27T00:14:41,196 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:41,196 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:41,196 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:41,196 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:41,196 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:41,196 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:41,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:41,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:41,448 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a3594c8b6f42452c8deb0ce0b39dfd1a 2024-11-27T00:14:41,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/edc38c35c3dd4aaf97240951daa2e5cd is 50, key is test_row_0/B:col10/1732666480043/Put/seqid=0 2024-11-27T00:14:41,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741964_1140 (size=12301) 2024-11-27T00:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T00:14:41,867 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/edc38c35c3dd4aaf97240951daa2e5cd 2024-11-27T00:14:41,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/26f7f6923e57429dbc9470c8bcf74ab3 is 50, key is test_row_0/C:col10/1732666480043/Put/seqid=0 2024-11-27T00:14:41,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741965_1141 (size=12301) 2024-11-27T00:14:42,121 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T00:14:42,285 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/26f7f6923e57429dbc9470c8bcf74ab3 2024-11-27T00:14:42,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/a3594c8b6f42452c8deb0ce0b39dfd1a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a3594c8b6f42452c8deb0ce0b39dfd1a 2024-11-27T00:14:42,300 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a3594c8b6f42452c8deb0ce0b39dfd1a, entries=150, sequenceid=557, filesize=12.0 K 2024-11-27T00:14:42,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/edc38c35c3dd4aaf97240951daa2e5cd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/edc38c35c3dd4aaf97240951daa2e5cd 2024-11-27T00:14:42,305 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/edc38c35c3dd4aaf97240951daa2e5cd, entries=150, sequenceid=557, filesize=12.0 K 2024-11-27T00:14:42,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/26f7f6923e57429dbc9470c8bcf74ab3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/26f7f6923e57429dbc9470c8bcf74ab3 2024-11-27T00:14:42,310 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/26f7f6923e57429dbc9470c8bcf74ab3, entries=150, sequenceid=557, filesize=12.0 K 2024-11-27T00:14:42,311 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for e7356b094232ddb421ff69892486ac78 in 1279ms, sequenceid=557, compaction requested=true 2024-11-27T00:14:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for e7356b094232ddb421ff69892486ac78: 2024-11-27T00:14:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-27T00:14:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-27T00:14:42,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-27T00:14:42,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8950 sec 2024-11-27T00:14:42,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.9000 sec 2024-11-27T00:14:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T00:14:42,565 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:49,371 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-27T00:14:49,371 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:14:49,371 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:49,372 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:49,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:49,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T00:14:49,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2772 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2657 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1196 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3588 rows 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1198 2024-11-27T00:14:49,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3593 rows 2024-11-27T00:14:49,373 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T00:14:49,373 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:14:49,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:49,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:49,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:14:49,378 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T00:14:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T00:14:49,385 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666489385"}]},"ts":"1732666489385"} 2024-11-27T00:14:49,388 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T00:14:49,388 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T00:14:49,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:14:49,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, UNASSIGN}] 2024-11-27T00:14:49,395 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, UNASSIGN 2024-11-27T00:14:49,396 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=e7356b094232ddb421ff69892486ac78, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:49,398 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33599 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=f036bc4986ea,33881,1732666453890, table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T00:14:49,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, UNASSIGN because future has completed 2024-11-27T00:14:49,400 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:14:49,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:14:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T00:14:49,561 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:49,561 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:14:49,563 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing e7356b094232ddb421ff69892486ac78, disabling compactions & flushes 2024-11-27T00:14:49,563 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:49,563 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:49,563 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. after waiting 0 ms 2024-11-27T00:14:49,563 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:49,563 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing e7356b094232ddb421ff69892486ac78 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T00:14:49,564 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=A 2024-11-27T00:14:49,564 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:49,564 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=B 2024-11-27T00:14:49,564 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:49,564 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7356b094232ddb421ff69892486ac78, store=C 2024-11-27T00:14:49,564 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:49,573 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/d8a48ba2be144c7b978a23e838f363b2 is 50, key is test_row_0/A:col10/1732666489365/Put/seqid=0 2024-11-27T00:14:49,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741966_1142 (size=12301) 2024-11-27T00:14:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T00:14:49,978 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/d8a48ba2be144c7b978a23e838f363b2 2024-11-27T00:14:49,986 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3a23314678424316a2ca6684e8ce0085 is 50, key is test_row_0/B:col10/1732666489365/Put/seqid=0 2024-11-27T00:14:49,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741967_1143 (size=12301) 2024-11-27T00:14:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T00:14:50,392 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3a23314678424316a2ca6684e8ce0085 2024-11-27T00:14:50,405 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/625ea76cf0e74d7a915faac8622f296d is 50, key is test_row_0/C:col10/1732666489365/Put/seqid=0 2024-11-27T00:14:50,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741968_1144 (size=12301) 2024-11-27T00:14:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T00:14:50,810 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/625ea76cf0e74d7a915faac8622f296d 2024-11-27T00:14:50,821 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/A/d8a48ba2be144c7b978a23e838f363b2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d8a48ba2be144c7b978a23e838f363b2 2024-11-27T00:14:50,827 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d8a48ba2be144c7b978a23e838f363b2, entries=150, sequenceid=565, filesize=12.0 K 2024-11-27T00:14:50,828 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/B/3a23314678424316a2ca6684e8ce0085 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3a23314678424316a2ca6684e8ce0085 2024-11-27T00:14:50,833 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3a23314678424316a2ca6684e8ce0085, entries=150, sequenceid=565, filesize=12.0 K 2024-11-27T00:14:50,834 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/.tmp/C/625ea76cf0e74d7a915faac8622f296d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/625ea76cf0e74d7a915faac8622f296d 2024-11-27T00:14:50,838 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/625ea76cf0e74d7a915faac8622f296d, entries=150, sequenceid=565, filesize=12.0 K 2024-11-27T00:14:50,839 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e7356b094232ddb421ff69892486ac78 in 1276ms, sequenceid=565, compaction requested=true 2024-11-27T00:14:50,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/458c746dbf6d459390093caf01561845, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/122285784d0349b6ad9b5c16fe4b6f32, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c3874a884ca94ce8b4ee91973a9b9194, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/120142262ea148b7bd7d62757eb3c19a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a34835943ed5404a931dae76cb67fc12, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d98424d7041c49898fd506e69c961738, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e06f6f0214134b15b89781194b03f9ef, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/07e6e54670a443dd9d9378abd59014f8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/964f7e27a6444a779de759fb29b77175, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b3c854faf2054d6b83559119398e4daa, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/24b363f3390f4473a3431ffae6c3375f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0084dbb3f42e40fdafb6720afa77be4f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/cebe18fc9ea04a94a61caad486c80788, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4729a3d3384b4234be73f58ae6765894, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b0e70f30d0784b48a97ea7531b11a673, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e9575f9dbc7e4ce298cd73f151d98309, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/ec4f010057a74acd8ce0f20c2f958177, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fa748dbe926d43f2bad4a8b0e50b2ebb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a1e50698e74f40a3a8f2eda1fb98b7d8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/829ad45265344824b6e03bb0cd0b71ee, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b99862942db944808153c82712a8cb92, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a0b13a42a7914366a23267fd4ddb5e87, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/324bc94975174d8f8e17876a92d803fd, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/eea0e03d1aac44abb414fbbda5caf994, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4766f11d102c4fbfaff3171e0fa7955f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/f667b274a06d46bb978d77e4fc8965f3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/981f8f0cbd93421d832a634e46e51031, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/286b4369b6854c6798118e91e5e49348, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/5f6484b65a7a47baaf5f702164856bee, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/292c004ad0434c669aa4efd6f5f317ce, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/00c43d15e19e453280ee78e6d2acc3ea, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a05550d5e606448f9daf743bf2f4710f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0ab9d98434ee46d39d32b00f64bad53a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/1dad3da8218240c38bc8a297fe1f0314, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/79e64a27ea4f49d6bb85bbeaea6e0fe9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/28df23f674604222a44dd3c1ebd92f26, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/03c79867dadb4eb5b6401d734cdfd882, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a2b68b4f59444b639d7b68e0204481f1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fdc42b6beefd419f98779ac225d9922f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0940f71f478547bab4722e8e11559eb2] to archive 2024-11-27T00:14:50,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:14:50,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/458c746dbf6d459390093caf01561845 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/458c746dbf6d459390093caf01561845 2024-11-27T00:14:50,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/122285784d0349b6ad9b5c16fe4b6f32 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/122285784d0349b6ad9b5c16fe4b6f32 2024-11-27T00:14:50,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c3874a884ca94ce8b4ee91973a9b9194 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c3874a884ca94ce8b4ee91973a9b9194 2024-11-27T00:14:50,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/120142262ea148b7bd7d62757eb3c19a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/120142262ea148b7bd7d62757eb3c19a 2024-11-27T00:14:50,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a34835943ed5404a931dae76cb67fc12 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a34835943ed5404a931dae76cb67fc12 2024-11-27T00:14:50,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d98424d7041c49898fd506e69c961738 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d98424d7041c49898fd506e69c961738 2024-11-27T00:14:50,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e06f6f0214134b15b89781194b03f9ef to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e06f6f0214134b15b89781194b03f9ef 2024-11-27T00:14:50,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/07e6e54670a443dd9d9378abd59014f8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/07e6e54670a443dd9d9378abd59014f8 2024-11-27T00:14:50,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/964f7e27a6444a779de759fb29b77175 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/964f7e27a6444a779de759fb29b77175 2024-11-27T00:14:50,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b3c854faf2054d6b83559119398e4daa to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b3c854faf2054d6b83559119398e4daa 2024-11-27T00:14:50,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/24b363f3390f4473a3431ffae6c3375f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/24b363f3390f4473a3431ffae6c3375f 2024-11-27T00:14:50,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0084dbb3f42e40fdafb6720afa77be4f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0084dbb3f42e40fdafb6720afa77be4f 2024-11-27T00:14:50,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/cebe18fc9ea04a94a61caad486c80788 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/cebe18fc9ea04a94a61caad486c80788 2024-11-27T00:14:50,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4729a3d3384b4234be73f58ae6765894 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4729a3d3384b4234be73f58ae6765894 2024-11-27T00:14:50,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b0e70f30d0784b48a97ea7531b11a673 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b0e70f30d0784b48a97ea7531b11a673 2024-11-27T00:14:50,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e9575f9dbc7e4ce298cd73f151d98309 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/e9575f9dbc7e4ce298cd73f151d98309 2024-11-27T00:14:50,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/ec4f010057a74acd8ce0f20c2f958177 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/ec4f010057a74acd8ce0f20c2f958177 2024-11-27T00:14:50,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fa748dbe926d43f2bad4a8b0e50b2ebb to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fa748dbe926d43f2bad4a8b0e50b2ebb 2024-11-27T00:14:50,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a1e50698e74f40a3a8f2eda1fb98b7d8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a1e50698e74f40a3a8f2eda1fb98b7d8 2024-11-27T00:14:50,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/829ad45265344824b6e03bb0cd0b71ee to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/829ad45265344824b6e03bb0cd0b71ee 2024-11-27T00:14:50,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b99862942db944808153c82712a8cb92 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/b99862942db944808153c82712a8cb92 2024-11-27T00:14:50,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a0b13a42a7914366a23267fd4ddb5e87 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a0b13a42a7914366a23267fd4ddb5e87 2024-11-27T00:14:50,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/324bc94975174d8f8e17876a92d803fd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/324bc94975174d8f8e17876a92d803fd 2024-11-27T00:14:50,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/eea0e03d1aac44abb414fbbda5caf994 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/eea0e03d1aac44abb414fbbda5caf994 2024-11-27T00:14:50,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4766f11d102c4fbfaff3171e0fa7955f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/4766f11d102c4fbfaff3171e0fa7955f 2024-11-27T00:14:50,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/f667b274a06d46bb978d77e4fc8965f3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/f667b274a06d46bb978d77e4fc8965f3 2024-11-27T00:14:50,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/981f8f0cbd93421d832a634e46e51031 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/981f8f0cbd93421d832a634e46e51031 2024-11-27T00:14:50,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/286b4369b6854c6798118e91e5e49348 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/286b4369b6854c6798118e91e5e49348 2024-11-27T00:14:50,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/5f6484b65a7a47baaf5f702164856bee to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/5f6484b65a7a47baaf5f702164856bee 2024-11-27T00:14:50,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/292c004ad0434c669aa4efd6f5f317ce to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/292c004ad0434c669aa4efd6f5f317ce 2024-11-27T00:14:50,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/00c43d15e19e453280ee78e6d2acc3ea to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/00c43d15e19e453280ee78e6d2acc3ea 2024-11-27T00:14:50,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a05550d5e606448f9daf743bf2f4710f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a05550d5e606448f9daf743bf2f4710f 2024-11-27T00:14:50,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0ab9d98434ee46d39d32b00f64bad53a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0ab9d98434ee46d39d32b00f64bad53a 2024-11-27T00:14:50,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/1dad3da8218240c38bc8a297fe1f0314 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/1dad3da8218240c38bc8a297fe1f0314 2024-11-27T00:14:50,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/79e64a27ea4f49d6bb85bbeaea6e0fe9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/79e64a27ea4f49d6bb85bbeaea6e0fe9 2024-11-27T00:14:50,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/28df23f674604222a44dd3c1ebd92f26 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/28df23f674604222a44dd3c1ebd92f26 2024-11-27T00:14:50,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/03c79867dadb4eb5b6401d734cdfd882 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/03c79867dadb4eb5b6401d734cdfd882 2024-11-27T00:14:50,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a2b68b4f59444b639d7b68e0204481f1 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a2b68b4f59444b639d7b68e0204481f1 2024-11-27T00:14:50,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fdc42b6beefd419f98779ac225d9922f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/fdc42b6beefd419f98779ac225d9922f 2024-11-27T00:14:50,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0940f71f478547bab4722e8e11559eb2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/0940f71f478547bab4722e8e11559eb2 2024-11-27T00:14:50,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/110c1e8804d14ec89790ddd53518dfad, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3fc2ae03ecad437f8dfc8aad872e5e8a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a36acf4dfd6543abb11903e947918470, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/74804aed5ca44fa68eee29f6590600aa, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c364a71e5c3942fdb1feb6d697e9bbe0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e7508018554f4fe3b32060c75b64f11a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ae080615910d4b2783690bcdb814746e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f2054bfb64d241e0a866f55bf1233e8f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e22d940fd344bf496ac91cacbe960f6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/043c03542fa24da9b2418520eac770ce, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/40f853961b6e45e2bb23f9a4d880e54d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ad5178f325df47ff93c80561b9acace9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/d863e3c695a7404ea4b68d76c935e326, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/acef2d147da247319338c2cea6479e15, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c345bfab3d1c4c1ca0f229bfd203bb04, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/526721fd5dc5439da338f5ecd5fc3f53, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f536d437572b4b63a2c957bccf808e18, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1b628aebcd114616a09b19688a56db6b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/38dc10e6fdb44ddc9bc6c41fa6b1155b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/743edfd38b1241a3a91d1d2af03495c5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c5f958023dd249f69cd55f7987fed4c5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf4919eb60af4ea984adef719c4189de, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a108c4986c8549c3afd32704fe7bf963, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/b84eaf052b9b4185a4e311fc6b03cab6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/075ce6b7d36f40608f0ca14778dbbb9a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3861eb5a4e87409481dc593f749cf8c2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9770604e5cc14cc3a527c4add37621bd, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/63ec36059dad4b66a2a526d980fb327a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ce89acc108484757ab2109f4ddce3293, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e69885f185854ba0bbbbc4b0878487ca, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ea8b3367dc9043c3a0ff73f0f2338581, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/493c1a76fee64e98835862bd17a83823, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/60b40d915e7a4d73a287c7d0f4e08525, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/7ed26768aa7b4f57a43cb8e8e610ccc8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/5075abc83a5e426c9745aeff3261cd04, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e953ff37c5e463ea32ab93dc880be15, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/4c76c8dd7fc7418e8b12ed7933f7fa95, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bba56710af954d5498c598e398b9cf7b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf0033714fbb4b2995b5344ad5b22bb9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1722f598210b496796c48bc6f1c22f3f] to archive 2024-11-27T00:14:50,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:14:50,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/110c1e8804d14ec89790ddd53518dfad to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/110c1e8804d14ec89790ddd53518dfad 2024-11-27T00:14:50,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3fc2ae03ecad437f8dfc8aad872e5e8a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3fc2ae03ecad437f8dfc8aad872e5e8a 2024-11-27T00:14:50,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a36acf4dfd6543abb11903e947918470 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a36acf4dfd6543abb11903e947918470 2024-11-27T00:14:50,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/74804aed5ca44fa68eee29f6590600aa to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/74804aed5ca44fa68eee29f6590600aa 2024-11-27T00:14:50,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c364a71e5c3942fdb1feb6d697e9bbe0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c364a71e5c3942fdb1feb6d697e9bbe0 2024-11-27T00:14:50,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e7508018554f4fe3b32060c75b64f11a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e7508018554f4fe3b32060c75b64f11a 2024-11-27T00:14:50,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ae080615910d4b2783690bcdb814746e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ae080615910d4b2783690bcdb814746e 2024-11-27T00:14:50,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f2054bfb64d241e0a866f55bf1233e8f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f2054bfb64d241e0a866f55bf1233e8f 2024-11-27T00:14:50,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e22d940fd344bf496ac91cacbe960f6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e22d940fd344bf496ac91cacbe960f6 2024-11-27T00:14:50,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/043c03542fa24da9b2418520eac770ce to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/043c03542fa24da9b2418520eac770ce 2024-11-27T00:14:50,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/40f853961b6e45e2bb23f9a4d880e54d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/40f853961b6e45e2bb23f9a4d880e54d 2024-11-27T00:14:50,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ad5178f325df47ff93c80561b9acace9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ad5178f325df47ff93c80561b9acace9 2024-11-27T00:14:50,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/d863e3c695a7404ea4b68d76c935e326 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/d863e3c695a7404ea4b68d76c935e326 2024-11-27T00:14:50,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/acef2d147da247319338c2cea6479e15 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/acef2d147da247319338c2cea6479e15 2024-11-27T00:14:50,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c345bfab3d1c4c1ca0f229bfd203bb04 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c345bfab3d1c4c1ca0f229bfd203bb04 2024-11-27T00:14:50,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/526721fd5dc5439da338f5ecd5fc3f53 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/526721fd5dc5439da338f5ecd5fc3f53 2024-11-27T00:14:50,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f536d437572b4b63a2c957bccf808e18 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/f536d437572b4b63a2c957bccf808e18 2024-11-27T00:14:50,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1b628aebcd114616a09b19688a56db6b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1b628aebcd114616a09b19688a56db6b 2024-11-27T00:14:50,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/38dc10e6fdb44ddc9bc6c41fa6b1155b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/38dc10e6fdb44ddc9bc6c41fa6b1155b 2024-11-27T00:14:50,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/743edfd38b1241a3a91d1d2af03495c5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/743edfd38b1241a3a91d1d2af03495c5 2024-11-27T00:14:50,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c5f958023dd249f69cd55f7987fed4c5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/c5f958023dd249f69cd55f7987fed4c5 2024-11-27T00:14:50,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf4919eb60af4ea984adef719c4189de to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf4919eb60af4ea984adef719c4189de 2024-11-27T00:14:50,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a108c4986c8549c3afd32704fe7bf963 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/a108c4986c8549c3afd32704fe7bf963 2024-11-27T00:14:50,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/b84eaf052b9b4185a4e311fc6b03cab6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/b84eaf052b9b4185a4e311fc6b03cab6 2024-11-27T00:14:50,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/075ce6b7d36f40608f0ca14778dbbb9a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/075ce6b7d36f40608f0ca14778dbbb9a 2024-11-27T00:14:50,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3861eb5a4e87409481dc593f749cf8c2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3861eb5a4e87409481dc593f749cf8c2 2024-11-27T00:14:50,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9770604e5cc14cc3a527c4add37621bd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9770604e5cc14cc3a527c4add37621bd 2024-11-27T00:14:50,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/63ec36059dad4b66a2a526d980fb327a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/63ec36059dad4b66a2a526d980fb327a 2024-11-27T00:14:50,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ce89acc108484757ab2109f4ddce3293 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ce89acc108484757ab2109f4ddce3293 2024-11-27T00:14:50,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e69885f185854ba0bbbbc4b0878487ca to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/e69885f185854ba0bbbbc4b0878487ca 2024-11-27T00:14:50,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ea8b3367dc9043c3a0ff73f0f2338581 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/ea8b3367dc9043c3a0ff73f0f2338581 2024-11-27T00:14:50,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/493c1a76fee64e98835862bd17a83823 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/493c1a76fee64e98835862bd17a83823 2024-11-27T00:14:50,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/60b40d915e7a4d73a287c7d0f4e08525 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/60b40d915e7a4d73a287c7d0f4e08525 2024-11-27T00:14:50,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/7ed26768aa7b4f57a43cb8e8e610ccc8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/7ed26768aa7b4f57a43cb8e8e610ccc8 2024-11-27T00:14:50,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/5075abc83a5e426c9745aeff3261cd04 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/5075abc83a5e426c9745aeff3261cd04 2024-11-27T00:14:50,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e953ff37c5e463ea32ab93dc880be15 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3e953ff37c5e463ea32ab93dc880be15 2024-11-27T00:14:50,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/4c76c8dd7fc7418e8b12ed7933f7fa95 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/4c76c8dd7fc7418e8b12ed7933f7fa95 2024-11-27T00:14:50,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bba56710af954d5498c598e398b9cf7b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bba56710af954d5498c598e398b9cf7b 2024-11-27T00:14:50,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf0033714fbb4b2995b5344ad5b22bb9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/bf0033714fbb4b2995b5344ad5b22bb9 2024-11-27T00:14:50,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1722f598210b496796c48bc6f1c22f3f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/1722f598210b496796c48bc6f1c22f3f 2024-11-27T00:14:50,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d5100c14d12c499895bcd561c4a3773b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/399f14ce91954ff7a3f80de0dcb6cdd4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/90f6a28765054f4f81b89226a2a3e152, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/8625d4e9b64f4875a79a684d8aebf4b9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/776b3acc8e4e424c9451ab7bc3e7ef11, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48faf2b3428b45f6a474e493a113178a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2b657ce194594e3b94dde45d685e3044, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/503760cb334f4063ad71c0012755063d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2f450cc7d86d430888a95929ea7648b2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48a84a22dba34a48b87b10a10159883a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/fb00401af284458aa5a02a7b54cdbedf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d0b7a206e7c74c20870e639b34148307, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/27c3cccac11d49cb9ee5617a055926da, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/36a0e9aa520e4490a99d229e69c0acbe, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/28cf16cabc8d40ed8215023caf235099, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1504c45d7743403ca45186c49787d819, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/eac04c53b8334726813e74a290459bcd, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/cd0067ee4ebf4cd7b91b93d193adb5b6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/6658cfca588c4c519235eb2fd03883eb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/18bf1eed84ea471d8463fc94dfd8f2bd, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ff66630c64874d8a867a8c0377b23a17, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f867a35a622048f98d87c164dc83d415, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ce9d8a3f17814d66b5e04727dd083d52, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f9e8d5978b1349f38b281a5844d8e17b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/b26e5463875040d9bea522a643939bf9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c2a2fdf2080f42e6ba90d8681283ec12, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bc43e51d842340bd9d28da687d65a9a1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1a39460579564ad1bbeb6f5202606152, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c8699b52dd99418290e617447d1ebe27, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/80272b0e71b84dd19c823cf51ece7f61, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/9dd28bf7d827468fb7dd99a74a8ead58, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/78cf4779902b44cc80ac8a66086f35b7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bf54a3b5220647ba8ed90f858ecb0190, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/a8d88e56ed944d87981dd65d57bdf796, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bee3cb4d103a4d05b4d75bc90d38165e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/971c6d3afd3f4146bc37adde41c2912a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/49273a2d2c534c9cb082f7bc069f0ef8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/482391a485db44f6b718f033b2db7975, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/3e3759acfb6d4a3aa9abd30bb633d1d3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/91708e578723421a848d13a3b8db47af] to archive 2024-11-27T00:14:50,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:14:50,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d5100c14d12c499895bcd561c4a3773b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d5100c14d12c499895bcd561c4a3773b 2024-11-27T00:14:50,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/399f14ce91954ff7a3f80de0dcb6cdd4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/399f14ce91954ff7a3f80de0dcb6cdd4 2024-11-27T00:14:50,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/90f6a28765054f4f81b89226a2a3e152 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/90f6a28765054f4f81b89226a2a3e152 2024-11-27T00:14:50,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/8625d4e9b64f4875a79a684d8aebf4b9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/8625d4e9b64f4875a79a684d8aebf4b9 2024-11-27T00:14:50,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/776b3acc8e4e424c9451ab7bc3e7ef11 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/776b3acc8e4e424c9451ab7bc3e7ef11 2024-11-27T00:14:50,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48faf2b3428b45f6a474e493a113178a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48faf2b3428b45f6a474e493a113178a 2024-11-27T00:14:50,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2b657ce194594e3b94dde45d685e3044 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2b657ce194594e3b94dde45d685e3044 2024-11-27T00:14:50,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/503760cb334f4063ad71c0012755063d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/503760cb334f4063ad71c0012755063d 2024-11-27T00:14:50,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2f450cc7d86d430888a95929ea7648b2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/2f450cc7d86d430888a95929ea7648b2 2024-11-27T00:14:50,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48a84a22dba34a48b87b10a10159883a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/48a84a22dba34a48b87b10a10159883a 2024-11-27T00:14:50,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/fb00401af284458aa5a02a7b54cdbedf to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/fb00401af284458aa5a02a7b54cdbedf 2024-11-27T00:14:50,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d0b7a206e7c74c20870e639b34148307 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/d0b7a206e7c74c20870e639b34148307 2024-11-27T00:14:50,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/27c3cccac11d49cb9ee5617a055926da to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/27c3cccac11d49cb9ee5617a055926da 2024-11-27T00:14:50,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/36a0e9aa520e4490a99d229e69c0acbe to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/36a0e9aa520e4490a99d229e69c0acbe 2024-11-27T00:14:50,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/28cf16cabc8d40ed8215023caf235099 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/28cf16cabc8d40ed8215023caf235099 2024-11-27T00:14:50,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1504c45d7743403ca45186c49787d819 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1504c45d7743403ca45186c49787d819 2024-11-27T00:14:50,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/eac04c53b8334726813e74a290459bcd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/eac04c53b8334726813e74a290459bcd 2024-11-27T00:14:50,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/cd0067ee4ebf4cd7b91b93d193adb5b6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/cd0067ee4ebf4cd7b91b93d193adb5b6 2024-11-27T00:14:50,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/6658cfca588c4c519235eb2fd03883eb to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/6658cfca588c4c519235eb2fd03883eb 2024-11-27T00:14:50,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/18bf1eed84ea471d8463fc94dfd8f2bd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/18bf1eed84ea471d8463fc94dfd8f2bd 2024-11-27T00:14:50,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ff66630c64874d8a867a8c0377b23a17 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ff66630c64874d8a867a8c0377b23a17 2024-11-27T00:14:50,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f867a35a622048f98d87c164dc83d415 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f867a35a622048f98d87c164dc83d415 2024-11-27T00:14:50,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ce9d8a3f17814d66b5e04727dd083d52 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/ce9d8a3f17814d66b5e04727dd083d52 2024-11-27T00:14:50,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f9e8d5978b1349f38b281a5844d8e17b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/f9e8d5978b1349f38b281a5844d8e17b 2024-11-27T00:14:50,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/b26e5463875040d9bea522a643939bf9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/b26e5463875040d9bea522a643939bf9 2024-11-27T00:14:50,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c2a2fdf2080f42e6ba90d8681283ec12 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c2a2fdf2080f42e6ba90d8681283ec12 2024-11-27T00:14:50,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bc43e51d842340bd9d28da687d65a9a1 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bc43e51d842340bd9d28da687d65a9a1 2024-11-27T00:14:50,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1a39460579564ad1bbeb6f5202606152 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/1a39460579564ad1bbeb6f5202606152 2024-11-27T00:14:50,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c8699b52dd99418290e617447d1ebe27 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/c8699b52dd99418290e617447d1ebe27 2024-11-27T00:14:50,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/80272b0e71b84dd19c823cf51ece7f61 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/80272b0e71b84dd19c823cf51ece7f61 2024-11-27T00:14:50,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/9dd28bf7d827468fb7dd99a74a8ead58 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/9dd28bf7d827468fb7dd99a74a8ead58 2024-11-27T00:14:50,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/78cf4779902b44cc80ac8a66086f35b7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/78cf4779902b44cc80ac8a66086f35b7 2024-11-27T00:14:50,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bf54a3b5220647ba8ed90f858ecb0190 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bf54a3b5220647ba8ed90f858ecb0190 2024-11-27T00:14:51,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/a8d88e56ed944d87981dd65d57bdf796 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/a8d88e56ed944d87981dd65d57bdf796 2024-11-27T00:14:51,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bee3cb4d103a4d05b4d75bc90d38165e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/bee3cb4d103a4d05b4d75bc90d38165e 2024-11-27T00:14:51,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/971c6d3afd3f4146bc37adde41c2912a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/971c6d3afd3f4146bc37adde41c2912a 2024-11-27T00:14:51,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/49273a2d2c534c9cb082f7bc069f0ef8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/49273a2d2c534c9cb082f7bc069f0ef8 2024-11-27T00:14:51,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/482391a485db44f6b718f033b2db7975 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/482391a485db44f6b718f033b2db7975 2024-11-27T00:14:51,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/3e3759acfb6d4a3aa9abd30bb633d1d3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/3e3759acfb6d4a3aa9abd30bb633d1d3 2024-11-27T00:14:51,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/91708e578723421a848d13a3b8db47af to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/91708e578723421a848d13a3b8db47af 2024-11-27T00:14:51,011 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/recovered.edits/568.seqid, newMaxSeqId=568, maxSeqId=1 2024-11-27T00:14:51,012 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. 2024-11-27T00:14:51,012 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for e7356b094232ddb421ff69892486ac78: Waiting for close lock at 1732666489562Running coprocessor pre-close hooks at 1732666489562Disabling compacts and flushes for region at 1732666489563 (+1 ms)Disabling writes for close at 1732666489563Obtaining lock to block concurrent updates at 1732666489563Preparing flush snapshotting stores in e7356b094232ddb421ff69892486ac78 at 1732666489563Finished memstore snapshotting TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732666489565 (+2 ms)Flushing stores of TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78. at 1732666489566 (+1 ms)Flushing e7356b094232ddb421ff69892486ac78/A: creating writer at 1732666489566Flushing e7356b094232ddb421ff69892486ac78/A: appending metadata at 1732666489573 (+7 ms)Flushing e7356b094232ddb421ff69892486ac78/A: closing flushed file at 1732666489573Flushing e7356b094232ddb421ff69892486ac78/B: creating writer at 1732666489983 (+410 ms)Flushing e7356b094232ddb421ff69892486ac78/B: appending metadata at 1732666489986 (+3 ms)Flushing e7356b094232ddb421ff69892486ac78/B: closing flushed file at 1732666489986Flushing e7356b094232ddb421ff69892486ac78/C: creating writer at 1732666490402 (+416 ms)Flushing e7356b094232ddb421ff69892486ac78/C: appending metadata at 1732666490405 (+3 ms)Flushing e7356b094232ddb421ff69892486ac78/C: closing flushed file at 1732666490405Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d84473d: reopening flushed file at 1732666490819 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47b9a65f: reopening flushed file at 1732666490828 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6870b3e6: reopening flushed file at 1732666490833 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e7356b094232ddb421ff69892486ac78 in 1276ms, sequenceid=565, compaction requested=true at 1732666490839 (+6 ms)Writing region close event to WAL at 1732666491008 (+169 ms)Running coprocessor post-close hooks at 1732666491011 (+3 ms)Closed at 1732666491012 (+1 ms) 2024-11-27T00:14:51,014 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:51,015 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=e7356b094232ddb421ff69892486ac78, regionState=CLOSED 2024-11-27T00:14:51,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:14:51,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-27T00:14:51,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure e7356b094232ddb421ff69892486ac78, server=f036bc4986ea,33881,1732666453890 in 1.6180 sec 2024-11-27T00:14:51,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-27T00:14:51,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7356b094232ddb421ff69892486ac78, UNASSIGN in 1.6260 sec 2024-11-27T00:14:51,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T00:14:51,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6330 sec 2024-11-27T00:14:51,027 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666491026"}]},"ts":"1732666491026"} 2024-11-27T00:14:51,028 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T00:14:51,029 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T00:14:51,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6510 sec 2024-11-27T00:14:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T00:14:51,546 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:51,556 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T00:14:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:51,565 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:51,567 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-27T00:14:51,571 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:51,575 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/recovered.edits] 2024-11-27T00:14:51,578 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a3594c8b6f42452c8deb0ce0b39dfd1a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/a3594c8b6f42452c8deb0ce0b39dfd1a 2024-11-27T00:14:51,580 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c880037ebd204468b01e358c1212514d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/c880037ebd204468b01e358c1212514d 2024-11-27T00:14:51,581 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d224d6ab4ea54987b2e5c1656933c470 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d224d6ab4ea54987b2e5c1656933c470 2024-11-27T00:14:51,582 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d8a48ba2be144c7b978a23e838f363b2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/A/d8a48ba2be144c7b978a23e838f363b2 2024-11-27T00:14:51,585 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3a23314678424316a2ca6684e8ce0085 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/3a23314678424316a2ca6684e8ce0085 2024-11-27T00:14:51,586 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/698a4d867ae94b5fb36b3d4c88b78177 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/698a4d867ae94b5fb36b3d4c88b78177 2024-11-27T00:14:51,587 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9d3335c6eadb485cb8ba10a1eb904a97 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/9d3335c6eadb485cb8ba10a1eb904a97 2024-11-27T00:14:51,588 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/edc38c35c3dd4aaf97240951daa2e5cd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/B/edc38c35c3dd4aaf97240951daa2e5cd 2024-11-27T00:14:51,591 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/156adc7af9f44de7adb128c8099a68ec to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/156adc7af9f44de7adb128c8099a68ec 2024-11-27T00:14:51,592 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/26f7f6923e57429dbc9470c8bcf74ab3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/26f7f6923e57429dbc9470c8bcf74ab3 2024-11-27T00:14:51,593 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/423cc8b8ac714585b728410e814c75a6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/423cc8b8ac714585b728410e814c75a6 2024-11-27T00:14:51,594 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/625ea76cf0e74d7a915faac8622f296d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/C/625ea76cf0e74d7a915faac8622f296d 2024-11-27T00:14:51,596 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/recovered.edits/568.seqid to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78/recovered.edits/568.seqid 2024-11-27T00:14:51,596 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e7356b094232ddb421ff69892486ac78 2024-11-27T00:14:51,597 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T00:14:51,601 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:51,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-27T00:14:51,610 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T00:14:51,639 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T00:14:51,641 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:51,641 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T00:14:51,642 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732666491641"}]},"ts":"9223372036854775807"} 2024-11-27T00:14:51,646 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T00:14:51,646 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e7356b094232ddb421ff69892486ac78, NAME => 'TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T00:14:51,646 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T00:14:51,646 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732666491646"}]},"ts":"9223372036854775807"} 2024-11-27T00:14:51,649 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T00:14:51,650 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:51,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-27T00:14:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-27T00:14:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T00:14:51,675 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:51,688 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=243 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;f036bc4986ea:33881-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_983676612_22 at /127.0.0.1:49452 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 248) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2431 (was 3018) 2024-11-27T00:14:51,698 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=243, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=2430 2024-11-27T00:14:51,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:14:51,700 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:14:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:51,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T00:14:51,703 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:51,703 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-27T00:14:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T00:14:51,704 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T00:14:51,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741969_1145 (size=1061) 2024-11-27T00:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T00:14:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T00:14:52,113 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 2024-11-27T00:14:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741970_1146 (size=53) 2024-11-27T00:14:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T00:14:52,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:52,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e769f58c73f73b55976db69d9226d87e, disabling compactions & flushes 2024-11-27T00:14:52,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:52,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:52,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. after waiting 0 ms 2024-11-27T00:14:52,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:52,520 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:52,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e769f58c73f73b55976db69d9226d87e: Waiting for close lock at 1732666492520Disabling compacts and flushes for region at 1732666492520Disabling writes for close at 1732666492520Writing region close event to WAL at 1732666492520Closed at 1732666492520 2024-11-27T00:14:52,523 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T00:14:52,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732666492523"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732666492523"}]},"ts":"1732666492523"} 2024-11-27T00:14:52,527 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T00:14:52,529 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T00:14:52,529 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666492529"}]},"ts":"1732666492529"} 2024-11-27T00:14:52,532 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T00:14:52,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, ASSIGN}] 2024-11-27T00:14:52,535 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, ASSIGN 2024-11-27T00:14:52,535 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, ASSIGN; state=OFFLINE, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=false 2024-11-27T00:14:52,686 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e769f58c73f73b55976db69d9226d87e, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:52,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, ASSIGN because future has completed 2024-11-27T00:14:52,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:14:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T00:14:52,845 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:52,845 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:14:52,845 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,846 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:52,846 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,846 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,848 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,851 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:52,852 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e769f58c73f73b55976db69d9226d87e columnFamilyName A 2024-11-27T00:14:52,852 DEBUG [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:52,853 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(327): Store=e769f58c73f73b55976db69d9226d87e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:52,853 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,855 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:52,855 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e769f58c73f73b55976db69d9226d87e columnFamilyName B 2024-11-27T00:14:52,855 DEBUG [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:52,856 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(327): Store=e769f58c73f73b55976db69d9226d87e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:52,856 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,857 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:52,857 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e769f58c73f73b55976db69d9226d87e columnFamilyName C 2024-11-27T00:14:52,857 DEBUG [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:52,858 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(327): Store=e769f58c73f73b55976db69d9226d87e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:52,858 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:52,858 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,858 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,859 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,860 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,860 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,860 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:14:52,861 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,863 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T00:14:52,864 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened e769f58c73f73b55976db69d9226d87e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63476900, jitterRate=-0.05412048101425171}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:14:52,864 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:52,864 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for e769f58c73f73b55976db69d9226d87e: Running coprocessor pre-open hook at 1732666492846Writing region info on filesystem at 1732666492846Initializing all the Stores at 1732666492847 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666492848 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666492848Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666492848Cleaning up temporary data from old regions at 1732666492860 (+12 ms)Running coprocessor post-open hooks at 1732666492864 (+4 ms)Region opened successfully at 1732666492864 2024-11-27T00:14:52,865 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., pid=36, masterSystemTime=1732666492841 2024-11-27T00:14:52,867 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:52,867 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:52,868 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e769f58c73f73b55976db69d9226d87e, regionState=OPEN, openSeqNum=2, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:52,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:14:52,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-27T00:14:52,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 in 182 msec 2024-11-27T00:14:52,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-27T00:14:52,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, ASSIGN in 339 msec 2024-11-27T00:14:52,875 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T00:14:52,875 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666492875"}]},"ts":"1732666492875"} 2024-11-27T00:14:52,877 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T00:14:52,877 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T00:14:52,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-27T00:14:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T00:14:53,856 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:53,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:53,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:53,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:53,864 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60847fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:53,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:53,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366f7c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:53,869 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:53,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:53,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:53,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:53,876 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T00:14:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f036bc4986ea,33599,1732666453115 2024-11-27T00:14:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@678171c4 2024-11-27T00:14:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T00:14:53,879 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T00:14:53,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:14:53,884 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:14:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T00:14:53,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741971_1147 (size=1097) 2024-11-27T00:14:54,304 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-27T00:14:54,304 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-27T00:14:54,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:14:54,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, REOPEN/MOVE}] 2024-11-27T00:14:54,321 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, REOPEN/MOVE 2024-11-27T00:14:54,322 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e769f58c73f73b55976db69d9226d87e, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:54,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, REOPEN/MOVE because future has completed 2024-11-27T00:14:54,324 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:14:54,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:14:54,477 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,477 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:14:54,477 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing e769f58c73f73b55976db69d9226d87e, disabling compactions & flushes 2024-11-27T00:14:54,477 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:54,477 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:54,477 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. after waiting 0 ms 2024-11-27T00:14:54,477 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:54,482 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T00:14:54,482 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:54,483 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for e769f58c73f73b55976db69d9226d87e: Waiting for close lock at 1732666494477Running coprocessor pre-close hooks at 1732666494477Disabling compacts and flushes for region at 1732666494477Disabling writes for close at 1732666494477Writing region close event to WAL at 1732666494478 (+1 ms)Running coprocessor post-close hooks at 1732666494482 (+4 ms)Closed at 1732666494482 2024-11-27T00:14:54,483 WARN [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: e769f58c73f73b55976db69d9226d87e to self. 2024-11-27T00:14:54,486 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,487 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e769f58c73f73b55976db69d9226d87e, regionState=CLOSED 2024-11-27T00:14:54,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:14:54,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-27T00:14:54,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 in 167 msec 2024-11-27T00:14:54,494 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, REOPEN/MOVE; state=CLOSED, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=true 2024-11-27T00:14:54,645 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e769f58c73f73b55976db69d9226d87e, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:54,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, REOPEN/MOVE because future has completed 2024-11-27T00:14:54,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:14:54,807 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:54,808 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:14:54,809 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,809 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:14:54,809 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,810 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,815 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,816 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:54,823 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e769f58c73f73b55976db69d9226d87e columnFamilyName A 2024-11-27T00:14:54,825 DEBUG [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:54,826 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(327): Store=e769f58c73f73b55976db69d9226d87e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:54,827 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,828 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:54,828 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e769f58c73f73b55976db69d9226d87e columnFamilyName B 2024-11-27T00:14:54,828 DEBUG [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:54,828 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(327): Store=e769f58c73f73b55976db69d9226d87e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:54,828 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,829 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:14:54,829 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e769f58c73f73b55976db69d9226d87e columnFamilyName C 2024-11-27T00:14:54,829 DEBUG [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:54,829 INFO [StoreOpener-e769f58c73f73b55976db69d9226d87e-1 {}] regionserver.HStore(327): Store=e769f58c73f73b55976db69d9226d87e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:14:54,830 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:54,830 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,830 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,831 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,832 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,832 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,832 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:14:54,834 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,834 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened e769f58c73f73b55976db69d9226d87e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70216570, jitterRate=0.046308428049087524}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:14:54,834 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,836 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for e769f58c73f73b55976db69d9226d87e: Running coprocessor pre-open hook at 1732666494810Writing region info on filesystem at 1732666494810Initializing all the Stores at 1732666494811 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666494811Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666494812 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666494812Cleaning up temporary data from old regions at 1732666494832 (+20 ms)Running coprocessor post-open hooks at 1732666494835 (+3 ms)Region opened successfully at 1732666494836 (+1 ms) 2024-11-27T00:14:54,837 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., pid=41, masterSystemTime=1732666494800 2024-11-27T00:14:54,839 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:54,839 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:54,839 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e769f58c73f73b55976db69d9226d87e, regionState=OPEN, openSeqNum=5, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:54,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:14:54,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-27T00:14:54,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 in 195 msec 2024-11-27T00:14:54,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-27T00:14:54,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, REOPEN/MOVE in 522 msec 2024-11-27T00:14:54,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-27T00:14:54,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 533 msec 2024-11-27T00:14:54,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 963 msec 2024-11-27T00:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-27T00:14:54,852 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:54,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,854 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a6c6562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,856 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffb4dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,858 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54499fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,860 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425ad5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,862 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10465ef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,863 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77752d2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bff8842, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,866 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af93599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,868 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d1835f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,870 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c211397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,872 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2215c99f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,873 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e66880, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,875 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41642a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,877 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3448488d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,879 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cf965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,881 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc3611c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:14:54,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:14:54,883 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:14:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:14:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:14:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5778a798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:14:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:14:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:14:54,884 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:14:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d54e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:14:54,885 DEBUG [Thread-722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,886 DEBUG [Thread-720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,886 DEBUG [Thread-726 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,886 DEBUG [Thread-728 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,886 DEBUG [Thread-737 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,888 DEBUG [Thread-731 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-27T00:14:54,891 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:54,892 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:54,892 DEBUG [Thread-735 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:54,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T00:14:54,894 DEBUG [Thread-724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,900 DEBUG [Thread-733 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:14:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:14:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:14:54,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:14:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:14:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:54,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T00:14:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:14:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:14:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:54,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:14:54,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:54,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666554941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:54,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666554943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666554948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:54,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666554945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272ef4f56d67b74b0fad98aa712aadf8d9_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666494914/Put/seqid=0 2024-11-27T00:14:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666554955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:54,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741972_1148 (size=12154) 2024-11-27T00:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T00:14:55,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T00:14:55,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:55,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666555065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666555065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666555065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666555065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666555073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T00:14:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T00:14:55,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666555273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666555274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666555274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666555274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666555284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T00:14:55,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:55,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:55,397 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272ef4f56d67b74b0fad98aa712aadf8d9_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272ef4f56d67b74b0fad98aa712aadf8d9_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:55,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/6722b4977c694abb86e33c781a6c3c29, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:55,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/6722b4977c694abb86e33c781a6c3c29 is 175, key is test_row_0/A:col10/1732666494914/Put/seqid=0 2024-11-27T00:14:55,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741973_1149 (size=30955) 2024-11-27T00:14:55,420 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/6722b4977c694abb86e33c781a6c3c29 2024-11-27T00:14:55,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/3d1921bff43f4c228a96f1101c50795e is 50, key is test_row_0/B:col10/1732666494914/Put/seqid=0 2024-11-27T00:14:55,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741974_1150 (size=12001) 2024-11-27T00:14:55,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/3d1921bff43f4c228a96f1101c50795e 2024-11-27T00:14:55,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/c9f008c40ffa47708721662835eadd4a is 50, key is test_row_0/C:col10/1732666494914/Put/seqid=0 2024-11-27T00:14:55,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T00:14:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T00:14:55,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741975_1151 (size=12001) 2024-11-27T00:14:55,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/c9f008c40ffa47708721662835eadd4a 2024-11-27T00:14:55,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/6722b4977c694abb86e33c781a6c3c29 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/6722b4977c694abb86e33c781a6c3c29 2024-11-27T00:14:55,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/6722b4977c694abb86e33c781a6c3c29, entries=150, sequenceid=16, filesize=30.2 K 2024-11-27T00:14:55,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/3d1921bff43f4c228a96f1101c50795e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/3d1921bff43f4c228a96f1101c50795e 2024-11-27T00:14:55,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/3d1921bff43f4c228a96f1101c50795e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T00:14:55,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/c9f008c40ffa47708721662835eadd4a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/c9f008c40ffa47708721662835eadd4a 2024-11-27T00:14:55,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/c9f008c40ffa47708721662835eadd4a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T00:14:55,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e769f58c73f73b55976db69d9226d87e in 659ms, sequenceid=16, compaction requested=false 2024-11-27T00:14:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T00:14:55,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:14:55,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666494936, max timestamp=1732666495586Num uniques 859; 2024-11-27T00:14:55,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:14:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:55,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-11-27T00:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:14:55,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112743dddb24d06d4120b88733d75f46a17a_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666494945/Put/seqid=0 2024-11-27T00:14:55,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741976_1152 (size=12154) 2024-11-27T00:14:55,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:55,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112743dddb24d06d4120b88733d75f46a17a_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112743dddb24d06d4120b88733d75f46a17a_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:55,643 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/ece0a89f2266469c88850c64f8660e82, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:55,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/ece0a89f2266469c88850c64f8660e82 is 175, key is test_row_0/A:col10/1732666494945/Put/seqid=0 2024-11-27T00:14:55,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666555640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666555647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666555650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666555650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666555652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741977_1153 (size=30955) 2024-11-27T00:14:55,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=38, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/ece0a89f2266469c88850c64f8660e82 2024-11-27T00:14:55,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T00:14:55,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:55,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/c773e5c61ac64617abc07abd2a592545 is 50, key is test_row_0/B:col10/1732666494945/Put/seqid=0 2024-11-27T00:14:55,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741978_1154 (size=12001) 2024-11-27T00:14:55,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/c773e5c61ac64617abc07abd2a592545 2024-11-27T00:14:55,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/8a6ec18c07f64b44a78afbe756c81da5 is 50, key is test_row_0/C:col10/1732666494945/Put/seqid=0 2024-11-27T00:14:55,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741979_1155 (size=12001) 2024-11-27T00:14:55,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/8a6ec18c07f64b44a78afbe756c81da5 2024-11-27T00:14:55,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/ece0a89f2266469c88850c64f8660e82 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/ece0a89f2266469c88850c64f8660e82 2024-11-27T00:14:55,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666555754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/ece0a89f2266469c88850c64f8660e82, entries=150, sequenceid=38, filesize=30.2 K 2024-11-27T00:14:55,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/c773e5c61ac64617abc07abd2a592545 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c773e5c61ac64617abc07abd2a592545 2024-11-27T00:14:55,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c773e5c61ac64617abc07abd2a592545, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T00:14:55,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/8a6ec18c07f64b44a78afbe756c81da5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/8a6ec18c07f64b44a78afbe756c81da5 2024-11-27T00:14:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666555764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666555764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666555764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666555764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:55,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/8a6ec18c07f64b44a78afbe756c81da5, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T00:14:55,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for e769f58c73f73b55976db69d9226d87e in 185ms, sequenceid=38, compaction requested=false 2024-11-27T00:14:55,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:14:55,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T00:14:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:55,823 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:14:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:14:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:14:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:14:55,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:55,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d29e206436be43938d54a68a00e86eb8_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666495649/Put/seqid=0 2024-11-27T00:14:55,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741980_1156 (size=12154) 2024-11-27T00:14:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:55,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:56,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666555995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666555999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666556001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666556002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666556004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T00:14:56,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666556113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666556114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666556114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666556114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666556114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:56,255 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d29e206436be43938d54a68a00e86eb8_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d29e206436be43938d54a68a00e86eb8_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:56,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/9ca88440217745fbb949debc94a4a0c7, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:56,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/9ca88440217745fbb949debc94a4a0c7 is 175, key is test_row_0/A:col10/1732666495649/Put/seqid=0 2024-11-27T00:14:56,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741981_1157 (size=30955) 2024-11-27T00:14:56,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666556324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666556324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666556324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666556325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666556325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,565 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T00:14:56,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666556634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666556634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666556634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666556634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666556635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:56,670 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/9ca88440217745fbb949debc94a4a0c7 2024-11-27T00:14:56,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/c884a1690c5c439e82bcb4e8ba9bacbf is 50, key is test_row_0/B:col10/1732666495649/Put/seqid=0 2024-11-27T00:14:56,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741982_1158 (size=12001) 2024-11-27T00:14:56,691 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/c884a1690c5c439e82bcb4e8ba9bacbf 2024-11-27T00:14:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/6e2c7d6ec80944c48c4a0693e35de249 is 50, key is test_row_0/C:col10/1732666495649/Put/seqid=0 2024-11-27T00:14:56,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741983_1159 (size=12001) 2024-11-27T00:14:56,714 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/6e2c7d6ec80944c48c4a0693e35de249 2024-11-27T00:14:56,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/9ca88440217745fbb949debc94a4a0c7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/9ca88440217745fbb949debc94a4a0c7 2024-11-27T00:14:56,733 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/9ca88440217745fbb949debc94a4a0c7, entries=150, sequenceid=53, filesize=30.2 K 2024-11-27T00:14:56,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/c884a1690c5c439e82bcb4e8ba9bacbf as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c884a1690c5c439e82bcb4e8ba9bacbf 2024-11-27T00:14:56,744 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c884a1690c5c439e82bcb4e8ba9bacbf, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T00:14:56,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/6e2c7d6ec80944c48c4a0693e35de249 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/6e2c7d6ec80944c48c4a0693e35de249 2024-11-27T00:14:56,757 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/6e2c7d6ec80944c48c4a0693e35de249, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T00:14:56,759 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e769f58c73f73b55976db69d9226d87e in 936ms, sequenceid=53, compaction requested=true 2024-11-27T00:14:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:14:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-27T00:14:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-27T00:14:56,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-27T00:14:56,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-11-27T00:14:56,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8780 sec 2024-11-27T00:14:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T00:14:57,044 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:57,045 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-27T00:14:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T00:14:57,049 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:57,050 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:57,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:57,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:14:57,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:14:57,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T00:14:57,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112749ebbde737f248bc8efe9db756e08041_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666495996/Put/seqid=0 2024-11-27T00:14:57,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666557176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666557177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666557177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666557183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666557184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741984_1160 (size=14594) 2024-11-27T00:14:57,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:57,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T00:14:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:57,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:57,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:57,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112749ebbde737f248bc8efe9db756e08041_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112749ebbde737f248bc8efe9db756e08041_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:14:57,210 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/2d9c079bef50432a891b05e002abb3dc, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:57,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/2d9c079bef50432a891b05e002abb3dc is 175, key is test_row_0/A:col10/1732666495996/Put/seqid=0 2024-11-27T00:14:57,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741985_1161 (size=39549) 2024-11-27T00:14:57,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=71, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/2d9c079bef50432a891b05e002abb3dc 2024-11-27T00:14:57,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/acbcc61d68dc4c2faf3314d857417e67 is 50, key is test_row_0/B:col10/1732666495996/Put/seqid=0 2024-11-27T00:14:57,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741986_1162 (size=12001) 2024-11-27T00:14:57,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/acbcc61d68dc4c2faf3314d857417e67 2024-11-27T00:14:57,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/9e1b18dc574a49a58e560fb320de50f9 is 50, key is test_row_0/C:col10/1732666495996/Put/seqid=0 2024-11-27T00:14:57,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741987_1163 (size=12001) 2024-11-27T00:14:57,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/9e1b18dc574a49a58e560fb320de50f9 2024-11-27T00:14:57,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/2d9c079bef50432a891b05e002abb3dc as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/2d9c079bef50432a891b05e002abb3dc 2024-11-27T00:14:57,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/2d9c079bef50432a891b05e002abb3dc, entries=200, sequenceid=71, filesize=38.6 K 2024-11-27T00:14:57,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/acbcc61d68dc4c2faf3314d857417e67 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/acbcc61d68dc4c2faf3314d857417e67 2024-11-27T00:14:57,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666557295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666557295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666557296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666557296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666557295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/acbcc61d68dc4c2faf3314d857417e67, entries=150, sequenceid=71, filesize=11.7 K 2024-11-27T00:14:57,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/9e1b18dc574a49a58e560fb320de50f9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/9e1b18dc574a49a58e560fb320de50f9 2024-11-27T00:14:57,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/9e1b18dc574a49a58e560fb320de50f9, entries=150, sequenceid=71, filesize=11.7 K 2024-11-27T00:14:57,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for e769f58c73f73b55976db69d9226d87e in 169ms, sequenceid=71, compaction requested=true 2024-11-27T00:14:57,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:14:57,318 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:57,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:14:57,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:57,319 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:14:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:57,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:14:57,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:57,321 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:57,321 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/A is initiating minor compaction (all files) 2024-11-27T00:14:57,321 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/A in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:57,321 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/6722b4977c694abb86e33c781a6c3c29, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/ece0a89f2266469c88850c64f8660e82, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/9ca88440217745fbb949debc94a4a0c7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/2d9c079bef50432a891b05e002abb3dc] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=129.3 K 2024-11-27T00:14:57,321 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:57,321 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/6722b4977c694abb86e33c781a6c3c29, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/ece0a89f2266469c88850c64f8660e82, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/9ca88440217745fbb949debc94a4a0c7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/2d9c079bef50432a891b05e002abb3dc] 2024-11-27T00:14:57,322 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:57,322 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/B is initiating minor compaction (all files) 2024-11-27T00:14:57,322 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/B in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:57,322 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/3d1921bff43f4c228a96f1101c50795e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c773e5c61ac64617abc07abd2a592545, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c884a1690c5c439e82bcb4e8ba9bacbf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/acbcc61d68dc4c2faf3314d857417e67] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=46.9 K 2024-11-27T00:14:57,322 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6722b4977c694abb86e33c781a6c3c29, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732666494912 2024-11-27T00:14:57,323 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d1921bff43f4c228a96f1101c50795e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732666494912 2024-11-27T00:14:57,323 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece0a89f2266469c88850c64f8660e82, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732666494945 2024-11-27T00:14:57,323 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c773e5c61ac64617abc07abd2a592545, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732666494945 2024-11-27T00:14:57,323 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca88440217745fbb949debc94a4a0c7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732666495619 2024-11-27T00:14:57,323 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c884a1690c5c439e82bcb4e8ba9bacbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732666495619 2024-11-27T00:14:57,324 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d9c079bef50432a891b05e002abb3dc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732666495996 2024-11-27T00:14:57,324 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting acbcc61d68dc4c2faf3314d857417e67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732666495996 2024-11-27T00:14:57,341 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:57,349 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411275078b9865e964c359604cd20240b21eb_e769f58c73f73b55976db69d9226d87e store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:57,358 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#B#compaction#145 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:57,359 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/74352411509442c39e7e64d927d0db63 is 50, key is test_row_0/B:col10/1732666495996/Put/seqid=0 2024-11-27T00:14:57,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T00:14:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:57,360 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:14:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:14:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:14:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:14:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T00:14:57,381 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411275078b9865e964c359604cd20240b21eb_e769f58c73f73b55976db69d9226d87e, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:57,381 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275078b9865e964c359604cd20240b21eb_e769f58c73f73b55976db69d9226d87e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:57,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741988_1164 (size=12139) 2024-11-27T00:14:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b7d58cf4fd444bc582ac25f9b29643c1_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666497171/Put/seqid=0 2024-11-27T00:14:57,411 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/74352411509442c39e7e64d927d0db63 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/74352411509442c39e7e64d927d0db63 2024-11-27T00:14:57,419 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/B of e769f58c73f73b55976db69d9226d87e into 74352411509442c39e7e64d927d0db63(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:57,419 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:14:57,419 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/B, priority=12, startTime=1732666497318; duration=0sec 2024-11-27T00:14:57,419 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:14:57,419 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:B 2024-11-27T00:14:57,420 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:14:57,423 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:14:57,423 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/C is initiating minor compaction (all files) 2024-11-27T00:14:57,423 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/C in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:57,423 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/c9f008c40ffa47708721662835eadd4a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/8a6ec18c07f64b44a78afbe756c81da5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/6e2c7d6ec80944c48c4a0693e35de249, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/9e1b18dc574a49a58e560fb320de50f9] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=46.9 K 2024-11-27T00:14:57,423 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f008c40ffa47708721662835eadd4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732666494912 2024-11-27T00:14:57,424 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6ec18c07f64b44a78afbe756c81da5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732666494945 2024-11-27T00:14:57,424 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2c7d6ec80944c48c4a0693e35de249, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732666495619 2024-11-27T00:14:57,425 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e1b18dc574a49a58e560fb320de50f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732666495996 2024-11-27T00:14:57,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741989_1165 (size=4469) 2024-11-27T00:14:57,432 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#A#compaction#144 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:57,433 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/d595a67a00564aada5eaad9d3326efd5 is 175, key is test_row_0/A:col10/1732666495996/Put/seqid=0 2024-11-27T00:14:57,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741990_1166 (size=12154) 2024-11-27T00:14:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:57,450 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b7d58cf4fd444bc582ac25f9b29643c1_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b7d58cf4fd444bc582ac25f9b29643c1_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:57,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/23db49a9a1be4b41b63a9a0784bbcf6a, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:57,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/23db49a9a1be4b41b63a9a0784bbcf6a is 175, key is test_row_0/A:col10/1732666497171/Put/seqid=0 2024-11-27T00:14:57,462 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:14:57,463 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/4c32eadc626749a0b618f5eb30b45b2a is 50, key is test_row_0/C:col10/1732666495996/Put/seqid=0 2024-11-27T00:14:57,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741991_1167 (size=31093) 2024-11-27T00:14:57,472 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/d595a67a00564aada5eaad9d3326efd5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d595a67a00564aada5eaad9d3326efd5 2024-11-27T00:14:57,479 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/A of e769f58c73f73b55976db69d9226d87e into d595a67a00564aada5eaad9d3326efd5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:57,479 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:14:57,479 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/A, priority=12, startTime=1732666497318; duration=0sec 2024-11-27T00:14:57,479 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:57,479 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:A 2024-11-27T00:14:57,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741992_1168 (size=30955) 2024-11-27T00:14:57,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741993_1169 (size=12139) 2024-11-27T00:14:57,493 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/4c32eadc626749a0b618f5eb30b45b2a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/4c32eadc626749a0b618f5eb30b45b2a 2024-11-27T00:14:57,501 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/C of e769f58c73f73b55976db69d9226d87e into 4c32eadc626749a0b618f5eb30b45b2a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:14:57,501 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:14:57,501 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/C, priority=12, startTime=1732666497319; duration=0sec 2024-11-27T00:14:57,501 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:14:57,501 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:C 2024-11-27T00:14:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:57,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:57,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666557522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666557524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666557525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666557533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666557533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666557644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666557644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666557644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666557644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666557645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T00:14:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666557854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666557854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666557854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666557854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666557854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:57,883 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/23db49a9a1be4b41b63a9a0784bbcf6a 2024-11-27T00:14:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/202a50046768410e84b32d2d79f0677e is 50, key is test_row_0/B:col10/1732666497171/Put/seqid=0 2024-11-27T00:14:57,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741994_1170 (size=12001) 2024-11-27T00:14:58,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666558164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666558164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666558164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666558164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666558164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T00:14:58,298 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/202a50046768410e84b32d2d79f0677e 2024-11-27T00:14:58,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/037a539b23d946f39fb18f206c74384a is 50, key is test_row_0/C:col10/1732666497171/Put/seqid=0 2024-11-27T00:14:58,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741995_1171 (size=12001) 2024-11-27T00:14:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666558674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666558674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666558674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666558675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666558675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:58,690 INFO [master/f036bc4986ea:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T00:14:58,690 INFO [master/f036bc4986ea:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T00:14:58,713 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/037a539b23d946f39fb18f206c74384a 2024-11-27T00:14:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/23db49a9a1be4b41b63a9a0784bbcf6a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/23db49a9a1be4b41b63a9a0784bbcf6a 2024-11-27T00:14:58,723 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/23db49a9a1be4b41b63a9a0784bbcf6a, entries=150, sequenceid=86, filesize=30.2 K 2024-11-27T00:14:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/202a50046768410e84b32d2d79f0677e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/202a50046768410e84b32d2d79f0677e 2024-11-27T00:14:58,742 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/202a50046768410e84b32d2d79f0677e, entries=150, sequenceid=86, filesize=11.7 K 2024-11-27T00:14:58,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/037a539b23d946f39fb18f206c74384a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/037a539b23d946f39fb18f206c74384a 2024-11-27T00:14:58,750 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/037a539b23d946f39fb18f206c74384a, entries=150, sequenceid=86, filesize=11.7 K 2024-11-27T00:14:58,752 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e769f58c73f73b55976db69d9226d87e in 1391ms, sequenceid=86, compaction requested=false 2024-11-27T00:14:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:14:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-27T00:14:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-27T00:14:58,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-27T00:14:58,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7030 sec 2024-11-27T00:14:58,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.7120 sec 2024-11-27T00:14:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T00:14:59,194 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:14:59,195 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:14:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-27T00:14:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T00:14:59,197 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:14:59,198 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:14:59,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:14:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T00:14:59,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T00:14:59,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:14:59,351 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:14:59,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:14:59,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:59,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:14:59,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:59,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:14:59,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:14:59,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112748c51b4ed1ce4b7db37afccbc2d6ff20_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666497523/Put/seqid=0 2024-11-27T00:14:59,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741996_1172 (size=12154) 2024-11-27T00:14:59,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:14:59,381 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112748c51b4ed1ce4b7db37afccbc2d6ff20_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112748c51b4ed1ce4b7db37afccbc2d6ff20_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:59,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/982da544f4f14dbda6b9fa5cad22da78, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:14:59,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/982da544f4f14dbda6b9fa5cad22da78 is 175, key is test_row_0/A:col10/1732666497523/Put/seqid=0 2024-11-27T00:14:59,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741997_1173 (size=30955) 2024-11-27T00:14:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T00:14:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:14:59,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:14:59,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666559702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666559702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666559706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666559707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666559707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,809 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/982da544f4f14dbda6b9fa5cad22da78 2024-11-27T00:14:59,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666559814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/53ffaab3d917457baa17688d7ba5174a is 50, key is test_row_0/B:col10/1732666497523/Put/seqid=0 2024-11-27T00:14:59,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666559820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741998_1174 (size=12001) 2024-11-27T00:14:59,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666559824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T00:14:59,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666559824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:14:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666559825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:14:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:14:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:14:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666560024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666560024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666560033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666560033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666560034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,224 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/53ffaab3d917457baa17688d7ba5174a 2024-11-27T00:15:00,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/323e0a52dd0e47e9a85cbb586f009a38 is 50, key is test_row_0/C:col10/1732666497523/Put/seqid=0 2024-11-27T00:15:00,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741999_1175 (size=12001) 2024-11-27T00:15:00,307 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/323e0a52dd0e47e9a85cbb586f009a38 2024-11-27T00:15:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/982da544f4f14dbda6b9fa5cad22da78 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/982da544f4f14dbda6b9fa5cad22da78 2024-11-27T00:15:00,324 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/982da544f4f14dbda6b9fa5cad22da78, entries=150, sequenceid=106, filesize=30.2 K 2024-11-27T00:15:00,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/53ffaab3d917457baa17688d7ba5174a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/53ffaab3d917457baa17688d7ba5174a 2024-11-27T00:15:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T00:15:00,338 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/53ffaab3d917457baa17688d7ba5174a, entries=150, sequenceid=106, filesize=11.7 K 2024-11-27T00:15:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/323e0a52dd0e47e9a85cbb586f009a38 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/323e0a52dd0e47e9a85cbb586f009a38 2024-11-27T00:15:00,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666560334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666560334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666560344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,348 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/323e0a52dd0e47e9a85cbb586f009a38, entries=150, sequenceid=106, filesize=11.7 K 2024-11-27T00:15:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666560344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,350 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e769f58c73f73b55976db69d9226d87e in 998ms, sequenceid=106, compaction requested=true 2024-11-27T00:15:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-27T00:15:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:00,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:15:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-27T00:15:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:00,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-27T00:15:00,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-11-27T00:15:00,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1600 sec 2024-11-27T00:15:00,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112783837a6d1013449eacb3f7de0e3d9806_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666499692/Put/seqid=0 2024-11-27T00:15:00,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742000_1176 (size=14594) 2024-11-27T00:15:00,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666560433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666560544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666560754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:00,820 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112783837a6d1013449eacb3f7de0e3d9806_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783837a6d1013449eacb3f7de0e3d9806_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:00,822 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/369792d571244baa893abcdc8f227f72, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:00,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/369792d571244baa893abcdc8f227f72 is 175, key is test_row_0/A:col10/1732666499692/Put/seqid=0 2024-11-27T00:15:00,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742001_1177 (size=39549) 2024-11-27T00:15:00,850 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/369792d571244baa893abcdc8f227f72 2024-11-27T00:15:00,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666560854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666560855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666560858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666560858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:00,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/675b8b754b11436e997ab638c4aaaa06 is 50, key is test_row_0/B:col10/1732666499692/Put/seqid=0 2024-11-27T00:15:00,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742002_1178 (size=12001) 2024-11-27T00:15:00,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/675b8b754b11436e997ab638c4aaaa06 2024-11-27T00:15:00,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/5ad532f8d90047a9abaf512a804815d1 is 50, key is test_row_0/C:col10/1732666499692/Put/seqid=0 2024-11-27T00:15:00,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742003_1179 (size=12001) 2024-11-27T00:15:01,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666561064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:01,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/5ad532f8d90047a9abaf512a804815d1 2024-11-27T00:15:01,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/369792d571244baa893abcdc8f227f72 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/369792d571244baa893abcdc8f227f72 2024-11-27T00:15:01,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/369792d571244baa893abcdc8f227f72, entries=200, sequenceid=122, filesize=38.6 K 2024-11-27T00:15:01,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/675b8b754b11436e997ab638c4aaaa06 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/675b8b754b11436e997ab638c4aaaa06 2024-11-27T00:15:01,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/675b8b754b11436e997ab638c4aaaa06, entries=150, sequenceid=122, filesize=11.7 K 2024-11-27T00:15:01,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/5ad532f8d90047a9abaf512a804815d1 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/5ad532f8d90047a9abaf512a804815d1 2024-11-27T00:15:01,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/5ad532f8d90047a9abaf512a804815d1, entries=150, sequenceid=122, filesize=11.7 K 2024-11-27T00:15:01,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e769f58c73f73b55976db69d9226d87e in 984ms, sequenceid=122, compaction requested=true 2024-11-27T00:15:01,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:01,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:01,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:01,334 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:01,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:01,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:01,334 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:01,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:01,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:01,335 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:01,335 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:01,336 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/A is initiating minor compaction (all files) 2024-11-27T00:15:01,336 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/B is initiating minor compaction (all files) 2024-11-27T00:15:01,336 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/B in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:01,336 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/A in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:01,336 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d595a67a00564aada5eaad9d3326efd5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/23db49a9a1be4b41b63a9a0784bbcf6a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/982da544f4f14dbda6b9fa5cad22da78, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/369792d571244baa893abcdc8f227f72] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=129.4 K 2024-11-27T00:15:01,336 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/74352411509442c39e7e64d927d0db63, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/202a50046768410e84b32d2d79f0677e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/53ffaab3d917457baa17688d7ba5174a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/675b8b754b11436e997ab638c4aaaa06] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=47.0 K 2024-11-27T00:15:01,336 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:01,336 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d595a67a00564aada5eaad9d3326efd5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/23db49a9a1be4b41b63a9a0784bbcf6a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/982da544f4f14dbda6b9fa5cad22da78, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/369792d571244baa893abcdc8f227f72] 2024-11-27T00:15:01,336 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 74352411509442c39e7e64d927d0db63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732666495996 2024-11-27T00:15:01,337 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d595a67a00564aada5eaad9d3326efd5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732666495996 2024-11-27T00:15:01,337 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 202a50046768410e84b32d2d79f0677e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732666497171 2024-11-27T00:15:01,337 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23db49a9a1be4b41b63a9a0784bbcf6a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732666497171 2024-11-27T00:15:01,337 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ffaab3d917457baa17688d7ba5174a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732666497522 2024-11-27T00:15:01,337 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 982da544f4f14dbda6b9fa5cad22da78, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732666497522 2024-11-27T00:15:01,338 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 675b8b754b11436e997ab638c4aaaa06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732666499692 2024-11-27T00:15:01,338 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 369792d571244baa893abcdc8f227f72, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732666499692 2024-11-27T00:15:01,347 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:01,348 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/5da564402b1b428780b89856420bc5fe is 50, key is test_row_0/B:col10/1732666499692/Put/seqid=0 2024-11-27T00:15:01,351 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T00:15:01,354 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:01,354 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:01,355 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127e6f942dbec474e0e9bb676b36863aca4_e769f58c73f73b55976db69d9226d87e store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:01,359 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127e6f942dbec474e0e9bb676b36863aca4_e769f58c73f73b55976db69d9226d87e, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:01,359 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e6f942dbec474e0e9bb676b36863aca4_e769f58c73f73b55976db69d9226d87e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-27T00:15:01,362 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T00:15:01,363 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:01,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:01,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742004_1180 (size=12275) 2024-11-27T00:15:01,372 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/5da564402b1b428780b89856420bc5fe as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/5da564402b1b428780b89856420bc5fe 2024-11-27T00:15:01,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742005_1181 (size=4469) 2024-11-27T00:15:01,381 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/B of e769f58c73f73b55976db69d9226d87e into 5da564402b1b428780b89856420bc5fe(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:01,381 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:01,381 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/B, priority=12, startTime=1732666501334; duration=0sec 2024-11-27T00:15:01,381 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:01,381 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:B 2024-11-27T00:15:01,381 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:01,384 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:01,384 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/C is initiating minor compaction (all files) 2024-11-27T00:15:01,384 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/C in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:01,384 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/4c32eadc626749a0b618f5eb30b45b2a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/037a539b23d946f39fb18f206c74384a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/323e0a52dd0e47e9a85cbb586f009a38, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/5ad532f8d90047a9abaf512a804815d1] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=47.0 K 2024-11-27T00:15:01,385 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c32eadc626749a0b618f5eb30b45b2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732666495996 2024-11-27T00:15:01,385 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 037a539b23d946f39fb18f206c74384a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732666497171 2024-11-27T00:15:01,385 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 323e0a52dd0e47e9a85cbb586f009a38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732666497522 2024-11-27T00:15:01,386 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ad532f8d90047a9abaf512a804815d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732666499692 2024-11-27T00:15:01,398 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#C#compaction#158 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:01,398 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/19c4cfe7ad814e01a6672d345a6b355c is 50, key is test_row_0/C:col10/1732666499692/Put/seqid=0 2024-11-27T00:15:01,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742006_1182 (size=12275) 2024-11-27T00:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T00:15:01,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T00:15:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:01,519 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:15:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:01,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ac65de07e24d49aa9289c175268a64bf_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666500419/Put/seqid=0 2024-11-27T00:15:01,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742007_1183 (size=12304) 2024-11-27T00:15:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:01,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:01,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666561633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T00:15:01,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666561744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:01,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:01,779 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#A#compaction#157 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:01,780 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8ae278e9d38f4e9d857edb26cbe1cb97 is 175, key is test_row_0/A:col10/1732666499692/Put/seqid=0 2024-11-27T00:15:01,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742008_1184 (size=31229) 2024-11-27T00:15:01,816 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/19c4cfe7ad814e01a6672d345a6b355c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/19c4cfe7ad814e01a6672d345a6b355c 2024-11-27T00:15:01,821 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/C of e769f58c73f73b55976db69d9226d87e into 19c4cfe7ad814e01a6672d345a6b355c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:01,821 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:01,821 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/C, priority=12, startTime=1732666501334; duration=0sec 2024-11-27T00:15:01,821 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:01,821 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:C 2024-11-27T00:15:01,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666561864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:01,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666561864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:01,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666561864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:01,865 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:01,865 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:01,866 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:01,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666561873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:01,879 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:01,936 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ac65de07e24d49aa9289c175268a64bf_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ac65de07e24d49aa9289c175268a64bf_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/3263ac2a44504da2a0bee020e1a96427, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/3263ac2a44504da2a0bee020e1a96427 is 175, key is test_row_0/A:col10/1732666500419/Put/seqid=0 2024-11-27T00:15:01,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742009_1185 (size=31105) 2024-11-27T00:15:01,943 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/3263ac2a44504da2a0bee020e1a96427 2024-11-27T00:15:01,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/cb079b7805ec4e8ca34a36022534f6a1 is 50, key is test_row_0/B:col10/1732666500419/Put/seqid=0 2024-11-27T00:15:01,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742010_1186 (size=12151) 2024-11-27T00:15:01,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666561954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T00:15:02,189 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8ae278e9d38f4e9d857edb26cbe1cb97 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8ae278e9d38f4e9d857edb26cbe1cb97 2024-11-27T00:15:02,195 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/A of e769f58c73f73b55976db69d9226d87e into 8ae278e9d38f4e9d857edb26cbe1cb97(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:02,195 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:02,195 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/A, priority=12, startTime=1732666501334; duration=0sec 2024-11-27T00:15:02,195 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:02,195 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:A 2024-11-27T00:15:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666562264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:02,358 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/cb079b7805ec4e8ca34a36022534f6a1 2024-11-27T00:15:02,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/ed3846d4420b48658e73e6e6ea18d5f2 is 50, key is test_row_0/C:col10/1732666500419/Put/seqid=0 2024-11-27T00:15:02,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742011_1187 (size=12151) 2024-11-27T00:15:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T00:15:02,773 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/ed3846d4420b48658e73e6e6ea18d5f2 2024-11-27T00:15:02,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666562774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:02,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/3263ac2a44504da2a0bee020e1a96427 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3263ac2a44504da2a0bee020e1a96427 2024-11-27T00:15:02,783 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3263ac2a44504da2a0bee020e1a96427, entries=150, sequenceid=139, filesize=30.4 K 2024-11-27T00:15:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/cb079b7805ec4e8ca34a36022534f6a1 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/cb079b7805ec4e8ca34a36022534f6a1 2024-11-27T00:15:02,788 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/cb079b7805ec4e8ca34a36022534f6a1, entries=150, sequenceid=139, filesize=11.9 K 2024-11-27T00:15:02,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/ed3846d4420b48658e73e6e6ea18d5f2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed3846d4420b48658e73e6e6ea18d5f2 2024-11-27T00:15:02,793 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed3846d4420b48658e73e6e6ea18d5f2, entries=150, sequenceid=139, filesize=11.9 K 2024-11-27T00:15:02,794 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e769f58c73f73b55976db69d9226d87e in 1276ms, sequenceid=139, compaction requested=false 2024-11-27T00:15:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-27T00:15:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-27T00:15:02,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-27T00:15:02,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4330 sec 2024-11-27T00:15:02,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4440 sec 2024-11-27T00:15:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T00:15:03,523 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:03,523 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-27T00:15:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T00:15:03,525 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:03,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:03,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T00:15:03,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T00:15:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:03,678 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:15:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:03,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:03,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127daaa7a5862e54df796294df4ab35fb0e_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666501613/Put/seqid=0 2024-11-27T00:15:03,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742012_1188 (size=12304) 2024-11-27T00:15:03,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:03,693 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127daaa7a5862e54df796294df4ab35fb0e_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127daaa7a5862e54df796294df4ab35fb0e_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:03,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/f341172f172946cfb736881403422b16, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:03,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/f341172f172946cfb736881403422b16 is 175, key is test_row_0/A:col10/1732666501613/Put/seqid=0 2024-11-27T00:15:03,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742013_1189 (size=31105) 2024-11-27T00:15:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:03,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:03,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666563836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T00:15:03,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666563874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666563874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:03,875 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:03,875 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:03,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666563884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:03,887 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:03,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666563894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:03,897 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:03,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666563944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:04,099 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/f341172f172946cfb736881403422b16 2024-11-27T00:15:04,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/9a2e80d0d3604b3aab024ab117420bb3 is 50, key is test_row_0/B:col10/1732666501613/Put/seqid=0 2024-11-27T00:15:04,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742014_1190 (size=12151) 2024-11-27T00:15:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T00:15:04,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666564153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666564474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:04,511 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/9a2e80d0d3604b3aab024ab117420bb3 2024-11-27T00:15:04,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/d496303db57c4ca6be60f0a667156345 is 50, key is test_row_0/C:col10/1732666501613/Put/seqid=0 2024-11-27T00:15:04,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742015_1191 (size=12151) 2024-11-27T00:15:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T00:15:04,922 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/d496303db57c4ca6be60f0a667156345 2024-11-27T00:15:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/f341172f172946cfb736881403422b16 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f341172f172946cfb736881403422b16 2024-11-27T00:15:04,932 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f341172f172946cfb736881403422b16, entries=150, sequenceid=157, filesize=30.4 K 2024-11-27T00:15:04,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/9a2e80d0d3604b3aab024ab117420bb3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/9a2e80d0d3604b3aab024ab117420bb3 2024-11-27T00:15:04,936 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/9a2e80d0d3604b3aab024ab117420bb3, entries=150, sequenceid=157, filesize=11.9 K 2024-11-27T00:15:04,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/d496303db57c4ca6be60f0a667156345 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d496303db57c4ca6be60f0a667156345 2024-11-27T00:15:04,942 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d496303db57c4ca6be60f0a667156345, entries=150, sequenceid=157, filesize=11.9 K 2024-11-27T00:15:04,943 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e769f58c73f73b55976db69d9226d87e in 1264ms, sequenceid=157, compaction requested=true 2024-11-27T00:15:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-27T00:15:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-27T00:15:04,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-27T00:15:04,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-27T00:15:04,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4230 sec 2024-11-27T00:15:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:04,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:15:04,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:04,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:04,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:04,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:04,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:04,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:04,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d69a6d334ac246d5833826ac0cecc55d_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666503822/Put/seqid=0 2024-11-27T00:15:05,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742016_1192 (size=14794) 2024-11-27T00:15:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666565045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:05,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666565154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:05,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666565364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:05,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:05,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d69a6d334ac246d5833826ac0cecc55d_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d69a6d334ac246d5833826ac0cecc55d_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:05,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/65d60de0d26d428ea615a5dfd42dbe05, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:05,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/65d60de0d26d428ea615a5dfd42dbe05 is 175, key is test_row_0/A:col10/1732666503822/Put/seqid=0 2024-11-27T00:15:05,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742017_1193 (size=39749) 2024-11-27T00:15:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T00:15:05,674 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:05,674 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:05,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666565673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-27T00:15:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T00:15:05,676 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:05,676 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:05,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T00:15:05,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/65d60de0d26d428ea615a5dfd42dbe05 2024-11-27T00:15:05,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/02236b5afb87411fa9e0c63f07c52bba is 50, key is test_row_0/B:col10/1732666503822/Put/seqid=0 2024-11-27T00:15:05,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742018_1194 (size=12151) 2024-11-27T00:15:05,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T00:15:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:05,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:05,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:05,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:05,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:05,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T00:15:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:05,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T00:15:06,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T00:15:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666566183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:06,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/02236b5afb87411fa9e0c63f07c52bba 2024-11-27T00:15:06,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/b5094b04130844b2b9a6955475c4aa52 is 50, key is test_row_0/C:col10/1732666503822/Put/seqid=0 2024-11-27T00:15:06,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742019_1195 (size=12151) 2024-11-27T00:15:06,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T00:15:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T00:15:06,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T00:15:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T00:15:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:06,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/b5094b04130844b2b9a6955475c4aa52 2024-11-27T00:15:06,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/65d60de0d26d428ea615a5dfd42dbe05 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/65d60de0d26d428ea615a5dfd42dbe05 2024-11-27T00:15:06,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/65d60de0d26d428ea615a5dfd42dbe05, entries=200, sequenceid=174, filesize=38.8 K 2024-11-27T00:15:06,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/02236b5afb87411fa9e0c63f07c52bba as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/02236b5afb87411fa9e0c63f07c52bba 2024-11-27T00:15:06,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/02236b5afb87411fa9e0c63f07c52bba, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T00:15:06,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/b5094b04130844b2b9a6955475c4aa52 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/b5094b04130844b2b9a6955475c4aa52 2024-11-27T00:15:06,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/b5094b04130844b2b9a6955475c4aa52, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T00:15:06,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e769f58c73f73b55976db69d9226d87e in 1681ms, sequenceid=174, compaction requested=true 2024-11-27T00:15:06,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:06,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:06,668 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:06,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:06,669 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:06,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:06,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:06,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:06,670 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:06,670 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/A is initiating minor compaction (all files) 2024-11-27T00:15:06,670 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/A in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,671 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8ae278e9d38f4e9d857edb26cbe1cb97, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3263ac2a44504da2a0bee020e1a96427, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f341172f172946cfb736881403422b16, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/65d60de0d26d428ea615a5dfd42dbe05] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=130.1 K 2024-11-27T00:15:06,671 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,671 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8ae278e9d38f4e9d857edb26cbe1cb97, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3263ac2a44504da2a0bee020e1a96427, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f341172f172946cfb736881403422b16, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/65d60de0d26d428ea615a5dfd42dbe05] 2024-11-27T00:15:06,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:06,671 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae278e9d38f4e9d857edb26cbe1cb97, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732666499692 2024-11-27T00:15:06,672 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3263ac2a44504da2a0bee020e1a96427, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732666500419 2024-11-27T00:15:06,672 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f341172f172946cfb736881403422b16, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732666501613 2024-11-27T00:15:06,672 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:06,672 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/B is initiating minor compaction (all files) 2024-11-27T00:15:06,672 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65d60de0d26d428ea615a5dfd42dbe05, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666503822 2024-11-27T00:15:06,672 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/B in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,672 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/5da564402b1b428780b89856420bc5fe, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/cb079b7805ec4e8ca34a36022534f6a1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/9a2e80d0d3604b3aab024ab117420bb3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/02236b5afb87411fa9e0c63f07c52bba] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=47.6 K 2024-11-27T00:15:06,673 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5da564402b1b428780b89856420bc5fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732666499692 2024-11-27T00:15:06,673 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting cb079b7805ec4e8ca34a36022534f6a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732666500419 2024-11-27T00:15:06,673 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a2e80d0d3604b3aab024ab117420bb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732666501613 2024-11-27T00:15:06,674 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 02236b5afb87411fa9e0c63f07c52bba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666503822 2024-11-27T00:15:06,689 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:06,694 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:06,695 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/7d26d33aa7da46c3ba19938270db2b07 is 50, key is test_row_0/B:col10/1732666503822/Put/seqid=0 2024-11-27T00:15:06,696 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411274597fa0b318045f8bd46f1f5deabcd97_e769f58c73f73b55976db69d9226d87e store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:06,702 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411274597fa0b318045f8bd46f1f5deabcd97_e769f58c73f73b55976db69d9226d87e, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:06,702 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274597fa0b318045f8bd46f1f5deabcd97_e769f58c73f73b55976db69d9226d87e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:06,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742020_1196 (size=12561) 2024-11-27T00:15:06,711 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/7d26d33aa7da46c3ba19938270db2b07 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/7d26d33aa7da46c3ba19938270db2b07 2024-11-27T00:15:06,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742021_1197 (size=4469) 2024-11-27T00:15:06,717 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#A#compaction#168 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:06,717 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/188f860ad4564aa585bbd58471c3e9e8 is 175, key is test_row_0/A:col10/1732666503822/Put/seqid=0 2024-11-27T00:15:06,724 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/B of e769f58c73f73b55976db69d9226d87e into 7d26d33aa7da46c3ba19938270db2b07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:06,724 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:06,724 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/B, priority=12, startTime=1732666506668; duration=0sec 2024-11-27T00:15:06,724 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:06,724 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:B 2024-11-27T00:15:06,724 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:06,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742022_1198 (size=31515) 2024-11-27T00:15:06,730 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:06,730 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/C is initiating minor compaction (all files) 2024-11-27T00:15:06,730 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/C in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,730 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/19c4cfe7ad814e01a6672d345a6b355c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed3846d4420b48658e73e6e6ea18d5f2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d496303db57c4ca6be60f0a667156345, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/b5094b04130844b2b9a6955475c4aa52] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=47.6 K 2024-11-27T00:15:06,731 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c4cfe7ad814e01a6672d345a6b355c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732666499692 2024-11-27T00:15:06,732 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ed3846d4420b48658e73e6e6ea18d5f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732666500419 2024-11-27T00:15:06,732 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d496303db57c4ca6be60f0a667156345, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732666501613 2024-11-27T00:15:06,733 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b5094b04130844b2b9a6955475c4aa52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666503822 2024-11-27T00:15:06,742 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:06,742 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/e4f7e117910d40c885fcb787f99adf0e is 50, key is test_row_0/C:col10/1732666503822/Put/seqid=0 2024-11-27T00:15:06,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742023_1199 (size=12561) 2024-11-27T00:15:06,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T00:15:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:06,749 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:15:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:06,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b6a096f2455448cb96921e06368f0b2e_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666505031/Put/seqid=0 2024-11-27T00:15:06,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742024_1200 (size=12304) 2024-11-27T00:15:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T00:15:07,131 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/188f860ad4564aa585bbd58471c3e9e8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/188f860ad4564aa585bbd58471c3e9e8 2024-11-27T00:15:07,136 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/A of e769f58c73f73b55976db69d9226d87e into 188f860ad4564aa585bbd58471c3e9e8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:07,136 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:07,136 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/A, priority=12, startTime=1732666506668; duration=0sec 2024-11-27T00:15:07,136 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:07,136 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:A 2024-11-27T00:15:07,152 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/e4f7e117910d40c885fcb787f99adf0e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/e4f7e117910d40c885fcb787f99adf0e 2024-11-27T00:15:07,161 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e769f58c73f73b55976db69d9226d87e/C of e769f58c73f73b55976db69d9226d87e into e4f7e117910d40c885fcb787f99adf0e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:07,161 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:07,161 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/C, priority=12, startTime=1732666506669; duration=0sec 2024-11-27T00:15:07,162 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:07,162 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:C 2024-11-27T00:15:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:07,175 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b6a096f2455448cb96921e06368f0b2e_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b6a096f2455448cb96921e06368f0b2e_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8c8d30c2285b448b8f1fdfd385c28894, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8c8d30c2285b448b8f1fdfd385c28894 is 175, key is test_row_0/A:col10/1732666505031/Put/seqid=0 2024-11-27T00:15:07,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742025_1201 (size=31105) 2024-11-27T00:15:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:07,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:07,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666567256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666567364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666567573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:07,582 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8c8d30c2285b448b8f1fdfd385c28894 2024-11-27T00:15:07,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/0eaa8763d0974437b9416a72167a35b7 is 50, key is test_row_0/B:col10/1732666505031/Put/seqid=0 2024-11-27T00:15:07,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742026_1202 (size=12151) 2024-11-27T00:15:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T00:15:07,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666567884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57316 deadline: 1732666567903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:07,906 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:07,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57302 deadline: 1732666567913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:07,918 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:07,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:07,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57346 deadline: 1732666567924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:07,931 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:07,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57362 deadline: 1732666567934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:07,937 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in e769f58c73f73b55976db69d9226d87e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:07,994 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/0eaa8763d0974437b9416a72167a35b7 2024-11-27T00:15:08,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/97bdebf48ad34db29b1a91c96226955b is 50, key is test_row_0/C:col10/1732666505031/Put/seqid=0 2024-11-27T00:15:08,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742027_1203 (size=12151) 2024-11-27T00:15:08,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666568394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:08,407 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/97bdebf48ad34db29b1a91c96226955b 2024-11-27T00:15:08,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8c8d30c2285b448b8f1fdfd385c28894 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8c8d30c2285b448b8f1fdfd385c28894 2024-11-27T00:15:08,418 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8c8d30c2285b448b8f1fdfd385c28894, entries=150, sequenceid=190, filesize=30.4 K 2024-11-27T00:15:08,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-27T00:15:08,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-27T00:15:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/0eaa8763d0974437b9416a72167a35b7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/0eaa8763d0974437b9416a72167a35b7 2024-11-27T00:15:08,431 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/0eaa8763d0974437b9416a72167a35b7, entries=150, sequenceid=190, filesize=11.9 K 2024-11-27T00:15:08,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/97bdebf48ad34db29b1a91c96226955b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/97bdebf48ad34db29b1a91c96226955b 2024-11-27T00:15:08,439 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/97bdebf48ad34db29b1a91c96226955b, entries=150, sequenceid=190, filesize=11.9 K 2024-11-27T00:15:08,440 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e769f58c73f73b55976db69d9226d87e in 1691ms, sequenceid=190, compaction requested=false 2024-11-27T00:15:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-27T00:15:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-27T00:15:08,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-27T00:15:08,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7650 sec 2024-11-27T00:15:08,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.7710 sec 2024-11-27T00:15:09,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:15:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:09,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:09,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:09,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112728b2ecb22e434221a0e3fe56db6c552f_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666507250/Put/seqid=0 2024-11-27T00:15:09,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742028_1204 (size=14794) 2024-11-27T00:15:09,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666569497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:09,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:09,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:09,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666569603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:09,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666569814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T00:15:09,854 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:09,854 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-27T00:15:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T00:15:09,863 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:09,864 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:09,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:09,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:09,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112728b2ecb22e434221a0e3fe56db6c552f_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112728b2ecb22e434221a0e3fe56db6c552f_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:09,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/dc2448a62b324da0b02d51cf0a77227f, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:09,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/dc2448a62b324da0b02d51cf0a77227f is 175, key is test_row_0/A:col10/1732666507250/Put/seqid=0 2024-11-27T00:15:09,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742029_1205 (size=39749) 2024-11-27T00:15:09,911 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/dc2448a62b324da0b02d51cf0a77227f 2024-11-27T00:15:09,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/1584d02e01cd4c338a796f3a449d483c is 50, key is test_row_0/B:col10/1732666507250/Put/seqid=0 2024-11-27T00:15:09,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742030_1206 (size=12151) 2024-11-27T00:15:09,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/1584d02e01cd4c338a796f3a449d483c 2024-11-27T00:15:09,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/1896c2b984e44fd68f44770a855bba90 is 50, key is test_row_0/C:col10/1732666507250/Put/seqid=0 2024-11-27T00:15:09,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742031_1207 (size=12151) 2024-11-27T00:15:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T00:15:10,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T00:15:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666570124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:10,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T00:15:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T00:15:10,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T00:15:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:10,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/1896c2b984e44fd68f44770a855bba90 2024-11-27T00:15:10,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/dc2448a62b324da0b02d51cf0a77227f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/dc2448a62b324da0b02d51cf0a77227f 2024-11-27T00:15:10,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/dc2448a62b324da0b02d51cf0a77227f, entries=200, sequenceid=210, filesize=38.8 K 2024-11-27T00:15:10,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/1584d02e01cd4c338a796f3a449d483c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/1584d02e01cd4c338a796f3a449d483c 2024-11-27T00:15:10,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/1584d02e01cd4c338a796f3a449d483c, entries=150, sequenceid=210, filesize=11.9 K 2024-11-27T00:15:10,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/1896c2b984e44fd68f44770a855bba90 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/1896c2b984e44fd68f44770a855bba90 2024-11-27T00:15:10,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/1896c2b984e44fd68f44770a855bba90, entries=150, sequenceid=210, filesize=11.9 K 2024-11-27T00:15:10,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e769f58c73f73b55976db69d9226d87e in 951ms, sequenceid=210, compaction requested=true 2024-11-27T00:15:10,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:10,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:10,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:10,368 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:10,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:10,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:10,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:10,368 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:10,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:10,369 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:10,369 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:10,369 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/A is initiating minor compaction (all files) 2024-11-27T00:15:10,369 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/B is initiating minor compaction (all files) 2024-11-27T00:15:10,369 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/A in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,369 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/B in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,369 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/7d26d33aa7da46c3ba19938270db2b07, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/0eaa8763d0974437b9416a72167a35b7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/1584d02e01cd4c338a796f3a449d483c] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=36.0 K 2024-11-27T00:15:10,369 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/188f860ad4564aa585bbd58471c3e9e8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8c8d30c2285b448b8f1fdfd385c28894, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/dc2448a62b324da0b02d51cf0a77227f] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=100.0 K 2024-11-27T00:15:10,369 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,369 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/188f860ad4564aa585bbd58471c3e9e8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8c8d30c2285b448b8f1fdfd385c28894, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/dc2448a62b324da0b02d51cf0a77227f] 2024-11-27T00:15:10,369 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d26d33aa7da46c3ba19938270db2b07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666503822 2024-11-27T00:15:10,370 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 188f860ad4564aa585bbd58471c3e9e8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666503822 2024-11-27T00:15:10,370 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eaa8763d0974437b9416a72167a35b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732666505031 2024-11-27T00:15:10,370 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1584d02e01cd4c338a796f3a449d483c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732666507244 2024-11-27T00:15:10,370 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8d30c2285b448b8f1fdfd385c28894, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732666505031 2024-11-27T00:15:10,370 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc2448a62b324da0b02d51cf0a77227f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732666507244 2024-11-27T00:15:10,376 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:10,377 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/8ce5fde0380241fa958a74197d8c25c8 is 50, key is test_row_0/B:col10/1732666507250/Put/seqid=0 2024-11-27T00:15:10,383 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:10,388 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112765dfe4f9abb1485ca31ae65378c9012f_e769f58c73f73b55976db69d9226d87e store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:10,389 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112765dfe4f9abb1485ca31ae65378c9012f_e769f58c73f73b55976db69d9226d87e, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:10,390 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112765dfe4f9abb1485ca31ae65378c9012f_e769f58c73f73b55976db69d9226d87e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:10,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742032_1208 (size=12663) 2024-11-27T00:15:10,397 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/8ce5fde0380241fa958a74197d8c25c8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/8ce5fde0380241fa958a74197d8c25c8 2024-11-27T00:15:10,402 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/B of e769f58c73f73b55976db69d9226d87e into 8ce5fde0380241fa958a74197d8c25c8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:10,403 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:10,403 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/B, priority=13, startTime=1732666510368; duration=0sec 2024-11-27T00:15:10,403 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:10,403 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:B 2024-11-27T00:15:10,403 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:10,406 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:10,406 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/C is initiating minor compaction (all files) 2024-11-27T00:15:10,406 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/C in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,406 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/e4f7e117910d40c885fcb787f99adf0e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/97bdebf48ad34db29b1a91c96226955b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/1896c2b984e44fd68f44770a855bba90] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=36.0 K 2024-11-27T00:15:10,407 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f7e117910d40c885fcb787f99adf0e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732666503822 2024-11-27T00:15:10,407 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 97bdebf48ad34db29b1a91c96226955b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732666505031 2024-11-27T00:15:10,408 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1896c2b984e44fd68f44770a855bba90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732666507244 2024-11-27T00:15:10,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742033_1209 (size=4469) 2024-11-27T00:15:10,416 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#A#compaction#178 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:10,417 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/d85b904d288341dd9678ed2005d827de is 175, key is test_row_0/A:col10/1732666507250/Put/seqid=0 2024-11-27T00:15:10,420 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:10,420 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/cd2fa0f80f0d420fb67d4bcc0c920a32 is 50, key is test_row_0/C:col10/1732666507250/Put/seqid=0 2024-11-27T00:15:10,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742034_1210 (size=31617) 2024-11-27T00:15:10,442 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/d85b904d288341dd9678ed2005d827de as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d85b904d288341dd9678ed2005d827de 2024-11-27T00:15:10,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742035_1211 (size=12663) 2024-11-27T00:15:10,450 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/A of e769f58c73f73b55976db69d9226d87e into d85b904d288341dd9678ed2005d827de(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:10,450 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:10,450 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/A, priority=13, startTime=1732666510368; duration=0sec 2024-11-27T00:15:10,450 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:10,450 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:A 2024-11-27T00:15:10,454 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/cd2fa0f80f0d420fb67d4bcc0c920a32 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cd2fa0f80f0d420fb67d4bcc0c920a32 2024-11-27T00:15:10,461 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/C of e769f58c73f73b55976db69d9226d87e into cd2fa0f80f0d420fb67d4bcc0c920a32(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:10,461 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:10,461 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/C, priority=13, startTime=1732666510368; duration=0sec 2024-11-27T00:15:10,461 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:10,461 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:C 2024-11-27T00:15:10,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T00:15:10,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:10,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:15:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T00:15:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d666ef3f8cfb499bb9fd8318b02b2c12_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666509494/Put/seqid=0 2024-11-27T00:15:10,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742036_1212 (size=12304) 2024-11-27T00:15:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:10,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:10,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666570683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666570794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:10,893 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d666ef3f8cfb499bb9fd8318b02b2c12_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d666ef3f8cfb499bb9fd8318b02b2c12_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/f63b0a47eec34d7dbe55c41fa02fdcee, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/f63b0a47eec34d7dbe55c41fa02fdcee is 175, key is test_row_0/A:col10/1732666509494/Put/seqid=0 2024-11-27T00:15:10,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742037_1213 (size=31105) 2024-11-27T00:15:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T00:15:11,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666571004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:11,300 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/f63b0a47eec34d7dbe55c41fa02fdcee 2024-11-27T00:15:11,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666571314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:11,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:11,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/69147ecd9ee04dcb9b24d42e71fe28d5 is 50, key is test_row_0/B:col10/1732666509494/Put/seqid=0 2024-11-27T00:15:11,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742038_1214 (size=12151) 2024-11-27T00:15:11,769 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/69147ecd9ee04dcb9b24d42e71fe28d5 2024-11-27T00:15:11,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/ed6b4bb1710942f19dd5eef7b11f1142 is 50, key is test_row_0/C:col10/1732666509494/Put/seqid=0 2024-11-27T00:15:11,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742039_1215 (size=12151) 2024-11-27T00:15:11,780 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/ed6b4bb1710942f19dd5eef7b11f1142 2024-11-27T00:15:11,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/f63b0a47eec34d7dbe55c41fa02fdcee as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f63b0a47eec34d7dbe55c41fa02fdcee 2024-11-27T00:15:11,790 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f63b0a47eec34d7dbe55c41fa02fdcee, entries=150, sequenceid=227, filesize=30.4 K 2024-11-27T00:15:11,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/69147ecd9ee04dcb9b24d42e71fe28d5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/69147ecd9ee04dcb9b24d42e71fe28d5 2024-11-27T00:15:11,795 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/69147ecd9ee04dcb9b24d42e71fe28d5, entries=150, sequenceid=227, filesize=11.9 K 2024-11-27T00:15:11,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/ed6b4bb1710942f19dd5eef7b11f1142 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed6b4bb1710942f19dd5eef7b11f1142 2024-11-27T00:15:11,800 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed6b4bb1710942f19dd5eef7b11f1142, entries=150, sequenceid=227, filesize=11.9 K 2024-11-27T00:15:11,801 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e769f58c73f73b55976db69d9226d87e in 1325ms, sequenceid=227, compaction requested=false 2024-11-27T00:15:11,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:11,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:11,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-27T00:15:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-27T00:15:11,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-27T00:15:11,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9380 sec 2024-11-27T00:15:11,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.9510 sec 2024-11-27T00:15:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:11,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:15:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:11,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:11,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a20dc7a589b140ef96f8824e02efbfbc_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666511824/Put/seqid=0 2024-11-27T00:15:11,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742040_1216 (size=14794) 2024-11-27T00:15:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666571867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:11,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666571973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T00:15:12,004 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:12,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-27T00:15:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T00:15:12,006 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:12,007 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:12,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T00:15:12,121 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T00:15:12,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T00:15:12,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:12,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666572183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:12,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:12,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:12,248 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a20dc7a589b140ef96f8824e02efbfbc_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a20dc7a589b140ef96f8824e02efbfbc_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:12,249 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/c72dc8cd569441c1b791932ea4d54c7b, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:12,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/c72dc8cd569441c1b791932ea4d54c7b is 175, key is test_row_0/A:col10/1732666511824/Put/seqid=0 2024-11-27T00:15:12,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742041_1217 (size=39749) 2024-11-27T00:15:12,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T00:15:12,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T00:15:12,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T00:15:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666572494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:12,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:12,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:12,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:12,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T00:15:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T00:15:12,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/c72dc8cd569441c1b791932ea4d54c7b 2024-11-27T00:15:12,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/6998b88825d24d3b8c905468555fdfb5 is 50, key is test_row_0/B:col10/1732666511824/Put/seqid=0 2024-11-27T00:15:12,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742042_1218 (size=12151) 2024-11-27T00:15:12,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/6998b88825d24d3b8c905468555fdfb5 2024-11-27T00:15:12,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/52fb8b21efb54e12a4b96822708ac634 is 50, key is test_row_0/C:col10/1732666511824/Put/seqid=0 2024-11-27T00:15:12,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742043_1219 (size=12151) 2024-11-27T00:15:12,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/52fb8b21efb54e12a4b96822708ac634 2024-11-27T00:15:12,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/c72dc8cd569441c1b791932ea4d54c7b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/c72dc8cd569441c1b791932ea4d54c7b 2024-11-27T00:15:12,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/c72dc8cd569441c1b791932ea4d54c7b, entries=200, sequenceid=246, filesize=38.8 K 2024-11-27T00:15:12,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/6998b88825d24d3b8c905468555fdfb5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/6998b88825d24d3b8c905468555fdfb5 2024-11-27T00:15:12,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/6998b88825d24d3b8c905468555fdfb5, entries=150, sequenceid=246, filesize=11.9 K 2024-11-27T00:15:12,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/52fb8b21efb54e12a4b96822708ac634 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/52fb8b21efb54e12a4b96822708ac634 2024-11-27T00:15:12,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/52fb8b21efb54e12a4b96822708ac634, entries=150, sequenceid=246, filesize=11.9 K 2024-11-27T00:15:12,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e769f58c73f73b55976db69d9226d87e in 885ms, sequenceid=246, compaction requested=true 2024-11-27T00:15:12,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:12,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:12,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:12,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:12,710 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:12,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:12,710 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:12,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:12,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:12,711 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:12,711 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:12,711 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/B is initiating minor compaction (all files) 2024-11-27T00:15:12,711 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/A is initiating minor compaction (all files) 2024-11-27T00:15:12,711 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/B in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,711 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/A in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,711 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d85b904d288341dd9678ed2005d827de, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f63b0a47eec34d7dbe55c41fa02fdcee, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/c72dc8cd569441c1b791932ea4d54c7b] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=100.1 K 2024-11-27T00:15:12,711 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/8ce5fde0380241fa958a74197d8c25c8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/69147ecd9ee04dcb9b24d42e71fe28d5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/6998b88825d24d3b8c905468555fdfb5] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=36.1 K 2024-11-27T00:15:12,712 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,712 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d85b904d288341dd9678ed2005d827de, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f63b0a47eec34d7dbe55c41fa02fdcee, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/c72dc8cd569441c1b791932ea4d54c7b] 2024-11-27T00:15:12,712 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d85b904d288341dd9678ed2005d827de, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732666507244 2024-11-27T00:15:12,715 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce5fde0380241fa958a74197d8c25c8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732666507244 2024-11-27T00:15:12,716 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f63b0a47eec34d7dbe55c41fa02fdcee, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732666509468 2024-11-27T00:15:12,716 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 69147ecd9ee04dcb9b24d42e71fe28d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732666509468 2024-11-27T00:15:12,716 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6998b88825d24d3b8c905468555fdfb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732666510678 2024-11-27T00:15:12,716 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c72dc8cd569441c1b791932ea4d54c7b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732666510676 2024-11-27T00:15:12,723 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:12,725 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411277fffd8f8ffcf4a90a4f2916266a1c438_e769f58c73f73b55976db69d9226d87e store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:12,726 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:12,726 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/f71fe45ba67e43f5a8a78b5fc5658f0a is 50, key is test_row_0/B:col10/1732666511824/Put/seqid=0 2024-11-27T00:15:12,728 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411277fffd8f8ffcf4a90a4f2916266a1c438_e769f58c73f73b55976db69d9226d87e, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:12,728 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277fffd8f8ffcf4a90a4f2916266a1c438_e769f58c73f73b55976db69d9226d87e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:12,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742044_1220 (size=12765) 2024-11-27T00:15:12,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742045_1221 (size=4469) 2024-11-27T00:15:12,750 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/f71fe45ba67e43f5a8a78b5fc5658f0a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f71fe45ba67e43f5a8a78b5fc5658f0a 2024-11-27T00:15:12,758 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/B of e769f58c73f73b55976db69d9226d87e into f71fe45ba67e43f5a8a78b5fc5658f0a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:12,758 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:12,758 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/B, priority=13, startTime=1732666512710; duration=0sec 2024-11-27T00:15:12,759 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:12,759 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:B 2024-11-27T00:15:12,759 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:12,760 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:12,760 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/C is initiating minor compaction (all files) 2024-11-27T00:15:12,760 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/C in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,760 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cd2fa0f80f0d420fb67d4bcc0c920a32, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed6b4bb1710942f19dd5eef7b11f1142, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/52fb8b21efb54e12a4b96822708ac634] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=36.1 K 2024-11-27T00:15:12,761 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2fa0f80f0d420fb67d4bcc0c920a32, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732666507244 2024-11-27T00:15:12,761 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ed6b4bb1710942f19dd5eef7b11f1142, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732666509468 2024-11-27T00:15:12,763 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 52fb8b21efb54e12a4b96822708ac634, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732666510678 2024-11-27T00:15:12,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T00:15:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:12,771 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T00:15:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:12,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:12,773 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:12,774 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/d117c6d83e31439b985d637851439995 is 50, key is test_row_0/C:col10/1732666511824/Put/seqid=0 2024-11-27T00:15:12,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742046_1222 (size=12765) 2024-11-27T00:15:12,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276d80b6e418264d96b2eac59d6c602e35_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666511852/Put/seqid=0 2024-11-27T00:15:12,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742047_1223 (size=12404) 2024-11-27T00:15:12,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:12,796 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276d80b6e418264d96b2eac59d6c602e35_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276d80b6e418264d96b2eac59d6c602e35_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:12,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/5351e385657747c2b79b11cfe07daad7, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:12,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/5351e385657747c2b79b11cfe07daad7 is 175, key is test_row_0/A:col10/1732666511852/Put/seqid=0 2024-11-27T00:15:12,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742048_1224 (size=31205) 2024-11-27T00:15:12,802 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/5351e385657747c2b79b11cfe07daad7 2024-11-27T00:15:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/638fe15fb9d94c15948a77167cbc6198 is 50, key is test_row_0/B:col10/1732666511852/Put/seqid=0 2024-11-27T00:15:12,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742049_1225 (size=12251) 2024-11-27T00:15:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:13,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:13,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666573084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:13,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T00:15:13,150 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#A#compaction#186 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:13,150 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/b06e8099371949bdae4f7931019a11b0 is 175, key is test_row_0/A:col10/1732666511824/Put/seqid=0 2024-11-27T00:15:13,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742050_1226 (size=31719) 2024-11-27T00:15:13,184 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/d117c6d83e31439b985d637851439995 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d117c6d83e31439b985d637851439995 2024-11-27T00:15:13,189 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/C of e769f58c73f73b55976db69d9226d87e into d117c6d83e31439b985d637851439995(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:13,189 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:13,189 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/C, priority=13, startTime=1732666512710; duration=0sec 2024-11-27T00:15:13,189 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:13,189 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:C 2024-11-27T00:15:13,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666573194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:13,220 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/638fe15fb9d94c15948a77167cbc6198 2024-11-27T00:15:13,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/d1db6ef951fb45d4af7734c6e1814e3a is 50, key is test_row_0/C:col10/1732666511852/Put/seqid=0 2024-11-27T00:15:13,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742051_1227 (size=12251) 2024-11-27T00:15:13,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666573404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:13,561 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/b06e8099371949bdae4f7931019a11b0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/b06e8099371949bdae4f7931019a11b0 2024-11-27T00:15:13,566 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/A of e769f58c73f73b55976db69d9226d87e into b06e8099371949bdae4f7931019a11b0(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:13,566 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:13,566 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/A, priority=13, startTime=1732666512710; duration=0sec 2024-11-27T00:15:13,566 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:13,566 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:A 2024-11-27T00:15:13,632 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/d1db6ef951fb45d4af7734c6e1814e3a 2024-11-27T00:15:13,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/5351e385657747c2b79b11cfe07daad7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5351e385657747c2b79b11cfe07daad7 2024-11-27T00:15:13,641 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5351e385657747c2b79b11cfe07daad7, entries=150, sequenceid=260, filesize=30.5 K 2024-11-27T00:15:13,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/638fe15fb9d94c15948a77167cbc6198 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/638fe15fb9d94c15948a77167cbc6198 2024-11-27T00:15:13,647 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/638fe15fb9d94c15948a77167cbc6198, entries=150, sequenceid=260, filesize=12.0 K 2024-11-27T00:15:13,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/d1db6ef951fb45d4af7734c6e1814e3a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d1db6ef951fb45d4af7734c6e1814e3a 2024-11-27T00:15:13,654 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d1db6ef951fb45d4af7734c6e1814e3a, entries=150, sequenceid=260, filesize=12.0 K 2024-11-27T00:15:13,655 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e769f58c73f73b55976db69d9226d87e in 883ms, sequenceid=260, compaction requested=false 2024-11-27T00:15:13,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:13,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:13,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-27T00:15:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-27T00:15:13,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-27T00:15:13,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6490 sec 2024-11-27T00:15:13,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.6530 sec 2024-11-27T00:15:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:13,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:15:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:13,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127311958b4e29641b8a4065d4aa449f369_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666513073/Put/seqid=0 2024-11-27T00:15:13,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742052_1228 (size=14994) 2024-11-27T00:15:13,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666573747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666573853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666574064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:14,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:14,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127311958b4e29641b8a4065d4aa449f369_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127311958b4e29641b8a4065d4aa449f369_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:14,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/bf8b2933c5a047068d252991670bb48e, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:14,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/bf8b2933c5a047068d252991670bb48e is 175, key is test_row_0/A:col10/1732666513073/Put/seqid=0 2024-11-27T00:15:14,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742053_1229 (size=39949) 2024-11-27T00:15:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T00:15:14,154 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:14,155 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-27T00:15:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T00:15:14,156 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:14,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:14,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T00:15:14,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:14,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:14,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666574374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:14,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:14,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:14,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:14,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:14,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:14,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T00:15:14,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/bf8b2933c5a047068d252991670bb48e 2024-11-27T00:15:14,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/f32dc174d5dd41bebaba3e9dee014521 is 50, key is test_row_0/B:col10/1732666513073/Put/seqid=0 2024-11-27T00:15:14,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742054_1230 (size=12301) 2024-11-27T00:15:14,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:14,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:14,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T00:15:14,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57286 deadline: 1732666574884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:14,897 INFO [Thread-733 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-733. 2024-11-27T00:15:14,897 DEBUG [Thread-733 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:14,897 DEBUG [Thread-733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:14,897 DEBUG [Thread-733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:14,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:14,899 INFO [Thread-731 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-731. 2024-11-27T00:15:14,899 DEBUG [Thread-731 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:14,899 DEBUG [Thread-731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:14,899 DEBUG [Thread-731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:14,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:14,900 INFO [Thread-737 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-737. 2024-11-27T00:15:14,900 DEBUG [Thread-737 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:14,900 DEBUG [Thread-737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:14,900 DEBUG [Thread-737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:14,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:14,902 INFO [Thread-735 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-735. 2024-11-27T00:15:14,902 DEBUG [Thread-735 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:14,902 DEBUG [Thread-735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:14,902 DEBUG [Thread-735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:14,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:14,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:14,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:14,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:14,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:14,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/f32dc174d5dd41bebaba3e9dee014521 2024-11-27T00:15:14,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/35fa7d64d26644be8560ad817825b8d3 is 50, key is test_row_0/C:col10/1732666513073/Put/seqid=0 2024-11-27T00:15:14,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742055_1231 (size=12301) 2024-11-27T00:15:15,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:15,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T00:15:15,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/35fa7d64d26644be8560ad817825b8d3 2024-11-27T00:15:15,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/bf8b2933c5a047068d252991670bb48e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/bf8b2933c5a047068d252991670bb48e 2024-11-27T00:15:15,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:15,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:15,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:15,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/bf8b2933c5a047068d252991670bb48e, entries=200, sequenceid=282, filesize=39.0 K 2024-11-27T00:15:15,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/f32dc174d5dd41bebaba3e9dee014521 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f32dc174d5dd41bebaba3e9dee014521 2024-11-27T00:15:15,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f32dc174d5dd41bebaba3e9dee014521, entries=150, sequenceid=282, filesize=12.0 K 2024-11-27T00:15:15,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/35fa7d64d26644be8560ad817825b8d3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/35fa7d64d26644be8560ad817825b8d3 2024-11-27T00:15:15,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/35fa7d64d26644be8560ad817825b8d3, entries=150, sequenceid=282, filesize=12.0 K 2024-11-27T00:15:15,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e769f58c73f73b55976db69d9226d87e in 1689ms, sequenceid=282, compaction requested=true 2024-11-27T00:15:15,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:15,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:15,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:15,404 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:15,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:15,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:15,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e769f58c73f73b55976db69d9226d87e:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:15,404 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:15,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:15,405 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:15,405 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:15,405 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/B is initiating minor compaction (all files) 2024-11-27T00:15:15,405 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/A is initiating minor compaction (all files) 2024-11-27T00:15:15,405 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/B in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,405 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/A in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,405 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f71fe45ba67e43f5a8a78b5fc5658f0a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/638fe15fb9d94c15948a77167cbc6198, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f32dc174d5dd41bebaba3e9dee014521] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=36.4 K 2024-11-27T00:15:15,405 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/b06e8099371949bdae4f7931019a11b0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5351e385657747c2b79b11cfe07daad7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/bf8b2933c5a047068d252991670bb48e] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=100.5 K 2024-11-27T00:15:15,405 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,405 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/b06e8099371949bdae4f7931019a11b0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5351e385657747c2b79b11cfe07daad7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/bf8b2933c5a047068d252991670bb48e] 2024-11-27T00:15:15,405 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f71fe45ba67e43f5a8a78b5fc5658f0a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732666510678 2024-11-27T00:15:15,406 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting b06e8099371949bdae4f7931019a11b0, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732666510678 2024-11-27T00:15:15,406 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 638fe15fb9d94c15948a77167cbc6198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732666511852 2024-11-27T00:15:15,406 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5351e385657747c2b79b11cfe07daad7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732666511852 2024-11-27T00:15:15,406 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f32dc174d5dd41bebaba3e9dee014521, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732666513073 2024-11-27T00:15:15,406 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf8b2933c5a047068d252991670bb48e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732666513073 2024-11-27T00:15:15,414 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#B#compaction#195 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:15,415 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/2aa1b6e1ff4c4634ab36c9f758ddc74e is 50, key is test_row_0/B:col10/1732666513073/Put/seqid=0 2024-11-27T00:15:15,416 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:15,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742056_1232 (size=13017) 2024-11-27T00:15:15,419 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127e7b21dbf769f4d59a46fb142807e32f5_e769f58c73f73b55976db69d9226d87e store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:15,449 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127e7b21dbf769f4d59a46fb142807e32f5_e769f58c73f73b55976db69d9226d87e, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:15,449 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e7b21dbf769f4d59a46fb142807e32f5_e769f58c73f73b55976db69d9226d87e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:15,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742057_1233 (size=4469) 2024-11-27T00:15:15,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-27T00:15:15,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,545 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T00:15:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:15,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274c530e1f265c45e28ac49d0b422cc806_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_0/A:col10/1732666513716/Put/seqid=0 2024-11-27T00:15:15,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742058_1234 (size=12454) 2024-11-27T00:15:15,829 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/2aa1b6e1ff4c4634ab36c9f758ddc74e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/2aa1b6e1ff4c4634ab36c9f758ddc74e 2024-11-27T00:15:15,838 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/B of e769f58c73f73b55976db69d9226d87e into 2aa1b6e1ff4c4634ab36c9f758ddc74e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:15,838 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:15,838 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/B, priority=13, startTime=1732666515404; duration=0sec 2024-11-27T00:15:15,838 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:15,838 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:B 2024-11-27T00:15:15,838 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:15,839 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:15,839 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): e769f58c73f73b55976db69d9226d87e/C is initiating minor compaction (all files) 2024-11-27T00:15:15,839 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e769f58c73f73b55976db69d9226d87e/C in TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:15,840 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d117c6d83e31439b985d637851439995, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d1db6ef951fb45d4af7734c6e1814e3a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/35fa7d64d26644be8560ad817825b8d3] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp, totalSize=36.4 K 2024-11-27T00:15:15,840 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d117c6d83e31439b985d637851439995, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732666510678 2024-11-27T00:15:15,840 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d1db6ef951fb45d4af7734c6e1814e3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732666511852 2024-11-27T00:15:15,841 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 35fa7d64d26644be8560ad817825b8d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732666513073 2024-11-27T00:15:15,855 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#C#compaction#198 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:15,855 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e769f58c73f73b55976db69d9226d87e#A#compaction#196 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:15,855 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/da7ededd1b7c475080d61d7a5ba2ae02 is 50, key is test_row_0/C:col10/1732666513073/Put/seqid=0 2024-11-27T00:15:15,856 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/3e8fb969b6434590a9f98571081e565e is 175, key is test_row_0/A:col10/1732666513073/Put/seqid=0 2024-11-27T00:15:15,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742059_1235 (size=13017) 2024-11-27T00:15:15,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742060_1236 (size=31971) 2024-11-27T00:15:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:15,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. as already flushing 2024-11-27T00:15:15,896 INFO [Thread-728 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-728. 2024-11-27T00:15:15,896 DEBUG [Thread-728 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:15,896 DEBUG [Thread-728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:15,896 DEBUG [Thread-728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:15,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:15,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:15,963 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274c530e1f265c45e28ac49d0b422cc806_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274c530e1f265c45e28ac49d0b422cc806_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:15,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/5ff9a25042bd43a3a12031d23d6b4a5b, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:15,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/5ff9a25042bd43a3a12031d23d6b4a5b is 175, key is test_row_0/A:col10/1732666513716/Put/seqid=0 2024-11-27T00:15:15,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742061_1237 (size=31255) 2024-11-27T00:15:16,273 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/3e8fb969b6434590a9f98571081e565e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3e8fb969b6434590a9f98571081e565e 2024-11-27T00:15:16,273 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/da7ededd1b7c475080d61d7a5ba2ae02 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/da7ededd1b7c475080d61d7a5ba2ae02 2024-11-27T00:15:16,279 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/C of e769f58c73f73b55976db69d9226d87e into da7ededd1b7c475080d61d7a5ba2ae02(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:16,279 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e769f58c73f73b55976db69d9226d87e/A of e769f58c73f73b55976db69d9226d87e into 3e8fb969b6434590a9f98571081e565e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:16,279 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:16,279 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:16,279 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/C, priority=13, startTime=1732666515404; duration=0sec 2024-11-27T00:15:16,279 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., storeName=e769f58c73f73b55976db69d9226d87e/A, priority=13, startTime=1732666515404; duration=0sec 2024-11-27T00:15:16,279 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:16,279 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:A 2024-11-27T00:15:16,279 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:16,279 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e769f58c73f73b55976db69d9226d87e:C 2024-11-27T00:15:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T00:15:16,381 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/5ff9a25042bd43a3a12031d23d6b4a5b 2024-11-27T00:15:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/d7217d5bfede4c4eaad7f6f79745829b is 50, key is test_row_0/B:col10/1732666513716/Put/seqid=0 2024-11-27T00:15:16,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742062_1238 (size=12301) 2024-11-27T00:15:16,797 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/d7217d5bfede4c4eaad7f6f79745829b 2024-11-27T00:15:16,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/17ffdccbbdce4b78b0bdd42240f00353 is 50, key is test_row_0/C:col10/1732666513716/Put/seqid=0 2024-11-27T00:15:16,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742063_1239 (size=12301) 2024-11-27T00:15:17,217 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/17ffdccbbdce4b78b0bdd42240f00353 2024-11-27T00:15:17,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/5ff9a25042bd43a3a12031d23d6b4a5b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5ff9a25042bd43a3a12031d23d6b4a5b 2024-11-27T00:15:17,233 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5ff9a25042bd43a3a12031d23d6b4a5b, entries=150, sequenceid=294, filesize=30.5 K 2024-11-27T00:15:17,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/d7217d5bfede4c4eaad7f6f79745829b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/d7217d5bfede4c4eaad7f6f79745829b 2024-11-27T00:15:17,240 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/d7217d5bfede4c4eaad7f6f79745829b, entries=150, sequenceid=294, filesize=12.0 K 2024-11-27T00:15:17,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/17ffdccbbdce4b78b0bdd42240f00353 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/17ffdccbbdce4b78b0bdd42240f00353 2024-11-27T00:15:17,244 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/17ffdccbbdce4b78b0bdd42240f00353, entries=150, sequenceid=294, filesize=12.0 K 2024-11-27T00:15:17,245 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for e769f58c73f73b55976db69d9226d87e in 1700ms, sequenceid=294, compaction requested=false 2024-11-27T00:15:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for e769f58c73f73b55976db69d9226d87e: 2024-11-27T00:15:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-27T00:15:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-27T00:15:17,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T00:15:17,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0890 sec 2024-11-27T00:15:17,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 3.0940 sec 2024-11-27T00:15:17,945 INFO [Thread-726 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-726. 2024-11-27T00:15:17,945 DEBUG [Thread-726 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:17,945 DEBUG [Thread-726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:17,946 DEBUG [Thread-726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:17,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:17,955 INFO [Thread-722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-722. 2024-11-27T00:15:17,955 DEBUG [Thread-722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:17,955 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:17,955 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:17,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:17,988 INFO [Thread-724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-724. 2024-11-27T00:15:17,988 INFO [Thread-720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-720. 2024-11-27T00:15:17,989 DEBUG [Thread-724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:17,989 DEBUG [Thread-720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:17,989 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:17,989 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:17,989 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:17,989 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:17,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:17,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T00:15:18,316 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:18,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T00:15:18,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-27T00:15:18,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3346 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3257 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1376 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4127 rows 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1385 2024-11-27T00:15:18,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4151 rows 2024-11-27T00:15:18,317 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T00:15:18,318 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:15:18,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:18,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:18,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:18,320 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T00:15:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T00:15:18,326 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666518325"}]},"ts":"1732666518325"} 2024-11-27T00:15:18,327 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T00:15:18,327 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T00:15:18,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:15:18,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, UNASSIGN}] 2024-11-27T00:15:18,330 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, UNASSIGN 2024-11-27T00:15:18,330 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=e769f58c73f73b55976db69d9226d87e, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:18,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, UNASSIGN because future has completed 2024-11-27T00:15:18,331 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:15:18,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:15:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T00:15:18,484 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:18,484 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:15:18,485 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing e769f58c73f73b55976db69d9226d87e, disabling compactions & flushes 2024-11-27T00:15:18,485 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:18,485 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:18,485 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. after waiting 0 ms 2024-11-27T00:15:18,485 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:18,485 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing e769f58c73f73b55976db69d9226d87e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T00:15:18,486 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=A 2024-11-27T00:15:18,486 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:18,486 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=B 2024-11-27T00:15:18,486 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:18,486 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e769f58c73f73b55976db69d9226d87e, store=C 2024-11-27T00:15:18,487 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:18,497 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270297d6e928ba47b8870f84f8dfb7aa13_e769f58c73f73b55976db69d9226d87e is 50, key is test_row_1/A:col10/1732666517985/Put/seqid=0 2024-11-27T00:15:18,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742064_1240 (size=9914) 2024-11-27T00:15:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T00:15:18,902 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:18,911 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270297d6e928ba47b8870f84f8dfb7aa13_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270297d6e928ba47b8870f84f8dfb7aa13_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:18,913 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8a2634f1920941b392c70e4274b4e849, store: [table=TestAcidGuarantees family=A region=e769f58c73f73b55976db69d9226d87e] 2024-11-27T00:15:18,913 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8a2634f1920941b392c70e4274b4e849 is 175, key is test_row_1/A:col10/1732666517985/Put/seqid=0 2024-11-27T00:15:18,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742065_1241 (size=22561) 2024-11-27T00:15:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T00:15:19,318 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8a2634f1920941b392c70e4274b4e849 2024-11-27T00:15:19,331 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/ee5c1fb4c5524736b97e58d1efd16733 is 50, key is test_row_1/B:col10/1732666517985/Put/seqid=0 2024-11-27T00:15:19,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742066_1242 (size=9857) 2024-11-27T00:15:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T00:15:19,737 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/ee5c1fb4c5524736b97e58d1efd16733 2024-11-27T00:15:19,752 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/cffeae4550f149ec980d610c5c91362b is 50, key is test_row_1/C:col10/1732666517985/Put/seqid=0 2024-11-27T00:15:19,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742067_1243 (size=9857) 2024-11-27T00:15:20,158 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/cffeae4550f149ec980d610c5c91362b 2024-11-27T00:15:20,169 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/A/8a2634f1920941b392c70e4274b4e849 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8a2634f1920941b392c70e4274b4e849 2024-11-27T00:15:20,174 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8a2634f1920941b392c70e4274b4e849, entries=100, sequenceid=305, filesize=22.0 K 2024-11-27T00:15:20,175 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/B/ee5c1fb4c5524736b97e58d1efd16733 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/ee5c1fb4c5524736b97e58d1efd16733 2024-11-27T00:15:20,180 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/ee5c1fb4c5524736b97e58d1efd16733, entries=100, sequenceid=305, filesize=9.6 K 2024-11-27T00:15:20,181 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/.tmp/C/cffeae4550f149ec980d610c5c91362b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cffeae4550f149ec980d610c5c91362b 2024-11-27T00:15:20,187 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cffeae4550f149ec980d610c5c91362b, entries=100, sequenceid=305, filesize=9.6 K 2024-11-27T00:15:20,188 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e769f58c73f73b55976db69d9226d87e in 1703ms, sequenceid=305, compaction requested=true 2024-11-27T00:15:20,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/6722b4977c694abb86e33c781a6c3c29, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/ece0a89f2266469c88850c64f8660e82, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/9ca88440217745fbb949debc94a4a0c7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/2d9c079bef50432a891b05e002abb3dc, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d595a67a00564aada5eaad9d3326efd5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/23db49a9a1be4b41b63a9a0784bbcf6a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/982da544f4f14dbda6b9fa5cad22da78, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/369792d571244baa893abcdc8f227f72, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8ae278e9d38f4e9d857edb26cbe1cb97, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3263ac2a44504da2a0bee020e1a96427, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f341172f172946cfb736881403422b16, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/65d60de0d26d428ea615a5dfd42dbe05, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/188f860ad4564aa585bbd58471c3e9e8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8c8d30c2285b448b8f1fdfd385c28894, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/dc2448a62b324da0b02d51cf0a77227f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d85b904d288341dd9678ed2005d827de, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f63b0a47eec34d7dbe55c41fa02fdcee, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/c72dc8cd569441c1b791932ea4d54c7b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/b06e8099371949bdae4f7931019a11b0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5351e385657747c2b79b11cfe07daad7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/bf8b2933c5a047068d252991670bb48e] to archive 2024-11-27T00:15:20,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:15:20,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/6722b4977c694abb86e33c781a6c3c29 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/6722b4977c694abb86e33c781a6c3c29 2024-11-27T00:15:20,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/ece0a89f2266469c88850c64f8660e82 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/ece0a89f2266469c88850c64f8660e82 2024-11-27T00:15:20,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/9ca88440217745fbb949debc94a4a0c7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/9ca88440217745fbb949debc94a4a0c7 2024-11-27T00:15:20,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/2d9c079bef50432a891b05e002abb3dc to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/2d9c079bef50432a891b05e002abb3dc 2024-11-27T00:15:20,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d595a67a00564aada5eaad9d3326efd5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d595a67a00564aada5eaad9d3326efd5 2024-11-27T00:15:20,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/23db49a9a1be4b41b63a9a0784bbcf6a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/23db49a9a1be4b41b63a9a0784bbcf6a 2024-11-27T00:15:20,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/982da544f4f14dbda6b9fa5cad22da78 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/982da544f4f14dbda6b9fa5cad22da78 2024-11-27T00:15:20,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/369792d571244baa893abcdc8f227f72 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/369792d571244baa893abcdc8f227f72 2024-11-27T00:15:20,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8ae278e9d38f4e9d857edb26cbe1cb97 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8ae278e9d38f4e9d857edb26cbe1cb97 2024-11-27T00:15:20,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3263ac2a44504da2a0bee020e1a96427 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3263ac2a44504da2a0bee020e1a96427 2024-11-27T00:15:20,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f341172f172946cfb736881403422b16 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f341172f172946cfb736881403422b16 2024-11-27T00:15:20,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/65d60de0d26d428ea615a5dfd42dbe05 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/65d60de0d26d428ea615a5dfd42dbe05 2024-11-27T00:15:20,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/188f860ad4564aa585bbd58471c3e9e8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/188f860ad4564aa585bbd58471c3e9e8 2024-11-27T00:15:20,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8c8d30c2285b448b8f1fdfd385c28894 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8c8d30c2285b448b8f1fdfd385c28894 2024-11-27T00:15:20,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/dc2448a62b324da0b02d51cf0a77227f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/dc2448a62b324da0b02d51cf0a77227f 2024-11-27T00:15:20,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d85b904d288341dd9678ed2005d827de to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/d85b904d288341dd9678ed2005d827de 2024-11-27T00:15:20,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f63b0a47eec34d7dbe55c41fa02fdcee to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/f63b0a47eec34d7dbe55c41fa02fdcee 2024-11-27T00:15:20,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/c72dc8cd569441c1b791932ea4d54c7b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/c72dc8cd569441c1b791932ea4d54c7b 2024-11-27T00:15:20,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/b06e8099371949bdae4f7931019a11b0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/b06e8099371949bdae4f7931019a11b0 2024-11-27T00:15:20,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5351e385657747c2b79b11cfe07daad7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5351e385657747c2b79b11cfe07daad7 2024-11-27T00:15:20,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/bf8b2933c5a047068d252991670bb48e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/bf8b2933c5a047068d252991670bb48e 2024-11-27T00:15:20,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/3d1921bff43f4c228a96f1101c50795e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c773e5c61ac64617abc07abd2a592545, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c884a1690c5c439e82bcb4e8ba9bacbf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/74352411509442c39e7e64d927d0db63, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/acbcc61d68dc4c2faf3314d857417e67, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/202a50046768410e84b32d2d79f0677e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/53ffaab3d917457baa17688d7ba5174a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/5da564402b1b428780b89856420bc5fe, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/675b8b754b11436e997ab638c4aaaa06, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/cb079b7805ec4e8ca34a36022534f6a1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/9a2e80d0d3604b3aab024ab117420bb3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/7d26d33aa7da46c3ba19938270db2b07, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/02236b5afb87411fa9e0c63f07c52bba, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/0eaa8763d0974437b9416a72167a35b7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/8ce5fde0380241fa958a74197d8c25c8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/1584d02e01cd4c338a796f3a449d483c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/69147ecd9ee04dcb9b24d42e71fe28d5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f71fe45ba67e43f5a8a78b5fc5658f0a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/6998b88825d24d3b8c905468555fdfb5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/638fe15fb9d94c15948a77167cbc6198, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f32dc174d5dd41bebaba3e9dee014521] to archive 2024-11-27T00:15:20,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:15:20,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/3d1921bff43f4c228a96f1101c50795e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/3d1921bff43f4c228a96f1101c50795e 2024-11-27T00:15:20,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c773e5c61ac64617abc07abd2a592545 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c773e5c61ac64617abc07abd2a592545 2024-11-27T00:15:20,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c884a1690c5c439e82bcb4e8ba9bacbf to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/c884a1690c5c439e82bcb4e8ba9bacbf 2024-11-27T00:15:20,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/74352411509442c39e7e64d927d0db63 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/74352411509442c39e7e64d927d0db63 2024-11-27T00:15:20,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/acbcc61d68dc4c2faf3314d857417e67 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/acbcc61d68dc4c2faf3314d857417e67 2024-11-27T00:15:20,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/202a50046768410e84b32d2d79f0677e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/202a50046768410e84b32d2d79f0677e 2024-11-27T00:15:20,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/53ffaab3d917457baa17688d7ba5174a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/53ffaab3d917457baa17688d7ba5174a 2024-11-27T00:15:20,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/5da564402b1b428780b89856420bc5fe to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/5da564402b1b428780b89856420bc5fe 2024-11-27T00:15:20,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/675b8b754b11436e997ab638c4aaaa06 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/675b8b754b11436e997ab638c4aaaa06 2024-11-27T00:15:20,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/cb079b7805ec4e8ca34a36022534f6a1 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/cb079b7805ec4e8ca34a36022534f6a1 2024-11-27T00:15:20,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/9a2e80d0d3604b3aab024ab117420bb3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/9a2e80d0d3604b3aab024ab117420bb3 2024-11-27T00:15:20,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/7d26d33aa7da46c3ba19938270db2b07 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/7d26d33aa7da46c3ba19938270db2b07 2024-11-27T00:15:20,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/02236b5afb87411fa9e0c63f07c52bba to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/02236b5afb87411fa9e0c63f07c52bba 2024-11-27T00:15:20,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/0eaa8763d0974437b9416a72167a35b7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/0eaa8763d0974437b9416a72167a35b7 2024-11-27T00:15:20,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/8ce5fde0380241fa958a74197d8c25c8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/8ce5fde0380241fa958a74197d8c25c8 2024-11-27T00:15:20,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/1584d02e01cd4c338a796f3a449d483c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/1584d02e01cd4c338a796f3a449d483c 2024-11-27T00:15:20,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/69147ecd9ee04dcb9b24d42e71fe28d5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/69147ecd9ee04dcb9b24d42e71fe28d5 2024-11-27T00:15:20,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f71fe45ba67e43f5a8a78b5fc5658f0a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f71fe45ba67e43f5a8a78b5fc5658f0a 2024-11-27T00:15:20,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/6998b88825d24d3b8c905468555fdfb5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/6998b88825d24d3b8c905468555fdfb5 2024-11-27T00:15:20,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/638fe15fb9d94c15948a77167cbc6198 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/638fe15fb9d94c15948a77167cbc6198 2024-11-27T00:15:20,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f32dc174d5dd41bebaba3e9dee014521 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/f32dc174d5dd41bebaba3e9dee014521 2024-11-27T00:15:20,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/c9f008c40ffa47708721662835eadd4a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/8a6ec18c07f64b44a78afbe756c81da5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/6e2c7d6ec80944c48c4a0693e35de249, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/4c32eadc626749a0b618f5eb30b45b2a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/9e1b18dc574a49a58e560fb320de50f9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/037a539b23d946f39fb18f206c74384a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/323e0a52dd0e47e9a85cbb586f009a38, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/19c4cfe7ad814e01a6672d345a6b355c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/5ad532f8d90047a9abaf512a804815d1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed3846d4420b48658e73e6e6ea18d5f2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d496303db57c4ca6be60f0a667156345, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/e4f7e117910d40c885fcb787f99adf0e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/b5094b04130844b2b9a6955475c4aa52, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/97bdebf48ad34db29b1a91c96226955b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cd2fa0f80f0d420fb67d4bcc0c920a32, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/1896c2b984e44fd68f44770a855bba90, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed6b4bb1710942f19dd5eef7b11f1142, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d117c6d83e31439b985d637851439995, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/52fb8b21efb54e12a4b96822708ac634, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d1db6ef951fb45d4af7734c6e1814e3a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/35fa7d64d26644be8560ad817825b8d3] to archive 2024-11-27T00:15:20,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:15:20,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/c9f008c40ffa47708721662835eadd4a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/c9f008c40ffa47708721662835eadd4a 2024-11-27T00:15:20,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/8a6ec18c07f64b44a78afbe756c81da5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/8a6ec18c07f64b44a78afbe756c81da5 2024-11-27T00:15:20,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/6e2c7d6ec80944c48c4a0693e35de249 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/6e2c7d6ec80944c48c4a0693e35de249 2024-11-27T00:15:20,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/4c32eadc626749a0b618f5eb30b45b2a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/4c32eadc626749a0b618f5eb30b45b2a 2024-11-27T00:15:20,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/9e1b18dc574a49a58e560fb320de50f9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/9e1b18dc574a49a58e560fb320de50f9 2024-11-27T00:15:20,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/037a539b23d946f39fb18f206c74384a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/037a539b23d946f39fb18f206c74384a 2024-11-27T00:15:20,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/323e0a52dd0e47e9a85cbb586f009a38 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/323e0a52dd0e47e9a85cbb586f009a38 2024-11-27T00:15:20,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/19c4cfe7ad814e01a6672d345a6b355c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/19c4cfe7ad814e01a6672d345a6b355c 2024-11-27T00:15:20,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/5ad532f8d90047a9abaf512a804815d1 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/5ad532f8d90047a9abaf512a804815d1 2024-11-27T00:15:20,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed3846d4420b48658e73e6e6ea18d5f2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed3846d4420b48658e73e6e6ea18d5f2 2024-11-27T00:15:20,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d496303db57c4ca6be60f0a667156345 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d496303db57c4ca6be60f0a667156345 2024-11-27T00:15:20,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/e4f7e117910d40c885fcb787f99adf0e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/e4f7e117910d40c885fcb787f99adf0e 2024-11-27T00:15:20,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/b5094b04130844b2b9a6955475c4aa52 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/b5094b04130844b2b9a6955475c4aa52 2024-11-27T00:15:20,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/97bdebf48ad34db29b1a91c96226955b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/97bdebf48ad34db29b1a91c96226955b 2024-11-27T00:15:20,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cd2fa0f80f0d420fb67d4bcc0c920a32 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cd2fa0f80f0d420fb67d4bcc0c920a32 2024-11-27T00:15:20,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/1896c2b984e44fd68f44770a855bba90 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/1896c2b984e44fd68f44770a855bba90 2024-11-27T00:15:20,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed6b4bb1710942f19dd5eef7b11f1142 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/ed6b4bb1710942f19dd5eef7b11f1142 2024-11-27T00:15:20,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d117c6d83e31439b985d637851439995 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d117c6d83e31439b985d637851439995 2024-11-27T00:15:20,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/52fb8b21efb54e12a4b96822708ac634 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/52fb8b21efb54e12a4b96822708ac634 2024-11-27T00:15:20,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d1db6ef951fb45d4af7734c6e1814e3a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/d1db6ef951fb45d4af7734c6e1814e3a 2024-11-27T00:15:20,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/35fa7d64d26644be8560ad817825b8d3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/35fa7d64d26644be8560ad817825b8d3 2024-11-27T00:15:20,305 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/recovered.edits/308.seqid, newMaxSeqId=308, maxSeqId=4 2024-11-27T00:15:20,305 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. 2024-11-27T00:15:20,306 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for e769f58c73f73b55976db69d9226d87e: Waiting for close lock at 1732666518485Running coprocessor pre-close hooks at 1732666518485Disabling compacts and flushes for region at 1732666518485Disabling writes for close at 1732666518485Obtaining lock to block concurrent updates at 1732666518485Preparing flush snapshotting stores in e769f58c73f73b55976db69d9226d87e at 1732666518485Finished memstore snapshotting TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732666518487 (+2 ms)Flushing stores of TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e. at 1732666518488 (+1 ms)Flushing e769f58c73f73b55976db69d9226d87e/A: creating writer at 1732666518488Flushing mob file e769f58c73f73b55976db69d9226d87e/A: appending metadata at 1732666518497 (+9 ms)Flushing mob file e769f58c73f73b55976db69d9226d87e/A: closing flushed file at 1732666518497Flushing e769f58c73f73b55976db69d9226d87e/A: appending metadata at 1732666518913 (+416 ms)Flushing e769f58c73f73b55976db69d9226d87e/A: closing flushed file at 1732666518913Flushing e769f58c73f73b55976db69d9226d87e/B: creating writer at 1732666519327 (+414 ms)Flushing e769f58c73f73b55976db69d9226d87e/B: appending metadata at 1732666519331 (+4 ms)Flushing e769f58c73f73b55976db69d9226d87e/B: closing flushed file at 1732666519331Flushing e769f58c73f73b55976db69d9226d87e/C: creating writer at 1732666519748 (+417 ms)Flushing e769f58c73f73b55976db69d9226d87e/C: appending metadata at 1732666519751 (+3 ms)Flushing e769f58c73f73b55976db69d9226d87e/C: closing flushed file at 1732666519751Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68a65cbc: reopening flushed file at 1732666520168 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64854205: reopening flushed file at 1732666520174 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12d2e771: reopening flushed file at 1732666520180 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e769f58c73f73b55976db69d9226d87e in 1703ms, sequenceid=305, compaction requested=true at 1732666520188 (+8 ms)Writing region close event to WAL at 1732666520300 (+112 ms)Running coprocessor post-close hooks at 1732666520305 (+5 ms)Closed at 1732666520305 2024-11-27T00:15:20,308 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,308 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=e769f58c73f73b55976db69d9226d87e, regionState=CLOSED 2024-11-27T00:15:20,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:15:20,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T00:15:20,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure e769f58c73f73b55976db69d9226d87e, server=f036bc4986ea,33881,1732666453890 in 1.9800 sec 2024-11-27T00:15:20,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-27T00:15:20,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e769f58c73f73b55976db69d9226d87e, UNASSIGN in 1.9850 sec 2024-11-27T00:15:20,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T00:15:20,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9880 sec 2024-11-27T00:15:20,320 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666520319"}]},"ts":"1732666520319"} 2024-11-27T00:15:20,321 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T00:15:20,321 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T00:15:20,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0010 sec 2024-11-27T00:15:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T00:15:20,475 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:20,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T00:15:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:20,477 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T00:15:20,478 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:20,481 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,485 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/recovered.edits] 2024-11-27T00:15:20,490 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3e8fb969b6434590a9f98571081e565e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/3e8fb969b6434590a9f98571081e565e 2024-11-27T00:15:20,492 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5ff9a25042bd43a3a12031d23d6b4a5b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/5ff9a25042bd43a3a12031d23d6b4a5b 2024-11-27T00:15:20,494 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8a2634f1920941b392c70e4274b4e849 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/A/8a2634f1920941b392c70e4274b4e849 2024-11-27T00:15:20,498 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/2aa1b6e1ff4c4634ab36c9f758ddc74e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/2aa1b6e1ff4c4634ab36c9f758ddc74e 2024-11-27T00:15:20,500 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/d7217d5bfede4c4eaad7f6f79745829b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/d7217d5bfede4c4eaad7f6f79745829b 2024-11-27T00:15:20,501 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/ee5c1fb4c5524736b97e58d1efd16733 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/B/ee5c1fb4c5524736b97e58d1efd16733 2024-11-27T00:15:20,503 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/17ffdccbbdce4b78b0bdd42240f00353 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/17ffdccbbdce4b78b0bdd42240f00353 2024-11-27T00:15:20,505 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cffeae4550f149ec980d610c5c91362b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/cffeae4550f149ec980d610c5c91362b 2024-11-27T00:15:20,506 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/da7ededd1b7c475080d61d7a5ba2ae02 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/C/da7ededd1b7c475080d61d7a5ba2ae02 2024-11-27T00:15:20,509 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/recovered.edits/308.seqid to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e/recovered.edits/308.seqid 2024-11-27T00:15:20,509 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,509 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T00:15:20,510 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T00:15:20,511 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T00:15:20,514 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270297d6e928ba47b8870f84f8dfb7aa13_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270297d6e928ba47b8870f84f8dfb7aa13_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,515 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112728b2ecb22e434221a0e3fe56db6c552f_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112728b2ecb22e434221a0e3fe56db6c552f_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,517 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272ef4f56d67b74b0fad98aa712aadf8d9_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272ef4f56d67b74b0fad98aa712aadf8d9_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,518 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127311958b4e29641b8a4065d4aa449f369_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127311958b4e29641b8a4065d4aa449f369_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,519 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112743dddb24d06d4120b88733d75f46a17a_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112743dddb24d06d4120b88733d75f46a17a_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,520 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112748c51b4ed1ce4b7db37afccbc2d6ff20_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112748c51b4ed1ce4b7db37afccbc2d6ff20_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,521 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112749ebbde737f248bc8efe9db756e08041_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112749ebbde737f248bc8efe9db756e08041_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,522 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274c530e1f265c45e28ac49d0b422cc806_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274c530e1f265c45e28ac49d0b422cc806_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,523 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276d80b6e418264d96b2eac59d6c602e35_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276d80b6e418264d96b2eac59d6c602e35_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,524 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783837a6d1013449eacb3f7de0e3d9806_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783837a6d1013449eacb3f7de0e3d9806_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,526 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a20dc7a589b140ef96f8824e02efbfbc_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a20dc7a589b140ef96f8824e02efbfbc_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,527 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ac65de07e24d49aa9289c175268a64bf_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ac65de07e24d49aa9289c175268a64bf_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,528 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b6a096f2455448cb96921e06368f0b2e_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b6a096f2455448cb96921e06368f0b2e_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,529 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b7d58cf4fd444bc582ac25f9b29643c1_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b7d58cf4fd444bc582ac25f9b29643c1_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,530 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d29e206436be43938d54a68a00e86eb8_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d29e206436be43938d54a68a00e86eb8_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,531 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d666ef3f8cfb499bb9fd8318b02b2c12_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d666ef3f8cfb499bb9fd8318b02b2c12_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,532 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d69a6d334ac246d5833826ac0cecc55d_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d69a6d334ac246d5833826ac0cecc55d_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,533 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127daaa7a5862e54df796294df4ab35fb0e_e769f58c73f73b55976db69d9226d87e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127daaa7a5862e54df796294df4ab35fb0e_e769f58c73f73b55976db69d9226d87e 2024-11-27T00:15:20,533 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T00:15:20,536 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:20,539 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T00:15:20,542 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T00:15:20,543 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:20,543 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T00:15:20,543 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732666520543"}]},"ts":"9223372036854775807"} 2024-11-27T00:15:20,546 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T00:15:20,546 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e769f58c73f73b55976db69d9226d87e, NAME => 'TestAcidGuarantees,,1732666491699.e769f58c73f73b55976db69d9226d87e.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T00:15:20,546 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T00:15:20,546 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732666520546"}]},"ts":"9223372036854775807"} 2024-11-27T00:15:20,548 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T00:15:20,549 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:20,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-11-27T00:15:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T00:15:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T00:15:20,584 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:20,595 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 243) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_983676612_22 at /127.0.0.1:49452 [Waiting for operation #252] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1879523407_22 at /127.0.0.1:34098 [Waiting for operation #229] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=386 (was 336) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2275 (was 2430) 2024-11-27T00:15:20,604 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=386, ProcessCount=11, AvailableMemoryMB=2274 2024-11-27T00:15:20,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:15:20,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:15:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:20,608 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T00:15:20,608 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:20,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-27T00:15:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T00:15:20,609 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T00:15:20,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742068_1244 (size=1061) 2024-11-27T00:15:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T00:15:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T00:15:21,017 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 2024-11-27T00:15:21,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742069_1245 (size=53) 2024-11-27T00:15:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T00:15:21,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:15:21,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 40e749124bfcc05793d8d1161b1bd9fa, disabling compactions & flushes 2024-11-27T00:15:21,426 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:21,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:21,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. after waiting 0 ms 2024-11-27T00:15:21,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:21,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:21,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 40e749124bfcc05793d8d1161b1bd9fa: Waiting for close lock at 1732666521426Disabling compacts and flushes for region at 1732666521426Disabling writes for close at 1732666521427 (+1 ms)Writing region close event to WAL at 1732666521427Closed at 1732666521427 2024-11-27T00:15:21,465 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T00:15:21,465 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732666521465"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732666521465"}]},"ts":"1732666521465"} 2024-11-27T00:15:21,467 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T00:15:21,468 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T00:15:21,468 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666521468"}]},"ts":"1732666521468"} 2024-11-27T00:15:21,470 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T00:15:21,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, ASSIGN}] 2024-11-27T00:15:21,472 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, ASSIGN 2024-11-27T00:15:21,473 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, ASSIGN; state=OFFLINE, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=false 2024-11-27T00:15:21,625 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=40e749124bfcc05793d8d1161b1bd9fa, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:21,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, ASSIGN because future has completed 2024-11-27T00:15:21,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:15:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T00:15:21,792 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:21,792 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:15:21,793 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,793 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:15:21,793 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,793 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,795 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,796 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:21,796 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40e749124bfcc05793d8d1161b1bd9fa columnFamilyName A 2024-11-27T00:15:21,797 DEBUG [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:21,797 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.HStore(327): Store=40e749124bfcc05793d8d1161b1bd9fa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:21,797 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,799 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:21,799 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40e749124bfcc05793d8d1161b1bd9fa columnFamilyName B 2024-11-27T00:15:21,799 DEBUG [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:21,800 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.HStore(327): Store=40e749124bfcc05793d8d1161b1bd9fa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:21,800 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,801 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:21,801 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40e749124bfcc05793d8d1161b1bd9fa columnFamilyName C 2024-11-27T00:15:21,801 DEBUG [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:21,802 INFO [StoreOpener-40e749124bfcc05793d8d1161b1bd9fa-1 {}] regionserver.HStore(327): Store=40e749124bfcc05793d8d1161b1bd9fa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:21,802 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:21,802 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,803 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,803 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,805 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,805 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,806 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:15:21,808 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,811 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T00:15:21,812 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 40e749124bfcc05793d8d1161b1bd9fa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70762324, jitterRate=0.05444079637527466}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:15:21,812 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:21,814 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 40e749124bfcc05793d8d1161b1bd9fa: Running coprocessor pre-open hook at 1732666521793Writing region info on filesystem at 1732666521793Initializing all the Stores at 1732666521794 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666521794Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666521794Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666521794Cleaning up temporary data from old regions at 1732666521805 (+11 ms)Running coprocessor post-open hooks at 1732666521812 (+7 ms)Region opened successfully at 1732666521814 (+2 ms) 2024-11-27T00:15:21,815 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., pid=67, masterSystemTime=1732666521784 2024-11-27T00:15:21,818 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:21,818 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:21,819 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=40e749124bfcc05793d8d1161b1bd9fa, regionState=OPEN, openSeqNum=2, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:21,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:15:21,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-27T00:15:21,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 in 193 msec 2024-11-27T00:15:21,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-27T00:15:21,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, ASSIGN in 354 msec 2024-11-27T00:15:21,827 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T00:15:21,827 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666521827"}]},"ts":"1732666521827"} 2024-11-27T00:15:21,828 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T00:15:21,829 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T00:15:21,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2230 sec 2024-11-27T00:15:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-27T00:15:22,765 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:22,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,766 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3e79ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1debf58d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,769 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,771 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T00:15:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is f036bc4986ea,33599,1732666453115 2024-11-27T00:15:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27accce3 2024-11-27T00:15:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T00:15:22,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T00:15:22,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8859531, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,777 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d146b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,779 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c067da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3d898f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,782 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d0efd1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,783 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c2f352d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,785 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac59d2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da7ec8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,789 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6ed58c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,790 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b32bd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,793 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3022c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,794 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35248724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ce8392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,798 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c10b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,800 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7e2220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,801 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a078dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,803 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a7a4f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,804 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5037e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:22,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:22,806 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fdbbd86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:22,807 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5365c66e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:22,808 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,808 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,808 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,808 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,808 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,809 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,809 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-27T00:15:22,810 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T00:15:22,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,811 DEBUG [Thread-1145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,811 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,811 DEBUG [Thread-1145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,811 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:22,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:22,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:22,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,816 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:22,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:22,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T00:15:22,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:22,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:22,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:22,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:22,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:22,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:22,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/ee04ba7bf43340c693f1269652dcd671 is 50, key is test_row_0/A:col10/1732666522816/Put/seqid=0 2024-11-27T00:15:22,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666582857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666582861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666582858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666582862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666582866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742070_1246 (size=12001) 2024-11-27T00:15:22,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/ee04ba7bf43340c693f1269652dcd671 2024-11-27T00:15:22,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/cb915c7d49704cbb8dd379b065661910 is 50, key is test_row_0/B:col10/1732666522816/Put/seqid=0 2024-11-27T00:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T00:15:22,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742071_1247 (size=12001) 2024-11-27T00:15:22,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/cb915c7d49704cbb8dd379b065661910 2024-11-27T00:15:22,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T00:15:22,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:22,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666582974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666582974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666582975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666582974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666582984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:22,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/ee48bd9bbf604c1b84c12b45b95970c1 is 50, key is test_row_0/C:col10/1732666522816/Put/seqid=0 2024-11-27T00:15:23,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742072_1248 (size=12001) 2024-11-27T00:15:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T00:15:23,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T00:15:23,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:23,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:23,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:23,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:23,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:23,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666583184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666583184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666583184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666583184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666583194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T00:15:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:23,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:23,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/ee48bd9bbf604c1b84c12b45b95970c1 2024-11-27T00:15:23,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/ee04ba7bf43340c693f1269652dcd671 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/ee04ba7bf43340c693f1269652dcd671 2024-11-27T00:15:23,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/ee04ba7bf43340c693f1269652dcd671, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T00:15:23,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/cb915c7d49704cbb8dd379b065661910 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/cb915c7d49704cbb8dd379b065661910 2024-11-27T00:15:23,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/cb915c7d49704cbb8dd379b065661910, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T00:15:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/ee48bd9bbf604c1b84c12b45b95970c1 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/ee48bd9bbf604c1b84c12b45b95970c1 2024-11-27T00:15:23,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/ee48bd9bbf604c1b84c12b45b95970c1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T00:15:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T00:15:23,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 40e749124bfcc05793d8d1161b1bd9fa in 611ms, sequenceid=13, compaction requested=false 2024-11-27T00:15:23,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T00:15:23,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:23,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:23,436 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:15:23,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:23,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:23,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/6eee32d620bf40c18538f420516d4bbf is 50, key is test_row_0/A:col10/1732666522854/Put/seqid=0 2024-11-27T00:15:23,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742073_1249 (size=12001) 2024-11-27T00:15:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:23,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:23,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666583501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666583502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666583503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666583503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666583505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666583614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666583614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666583614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666583614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666583824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666583824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666583824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666583824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:23,862 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/6eee32d620bf40c18538f420516d4bbf 2024-11-27T00:15:23,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/dfac0a77d38a4da89778d3ec1958a1d9 is 50, key is test_row_0/B:col10/1732666522854/Put/seqid=0 2024-11-27T00:15:23,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742074_1250 (size=12001) 2024-11-27T00:15:23,884 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/dfac0a77d38a4da89778d3ec1958a1d9 2024-11-27T00:15:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5ad39b944d6e46efb240229fdcb98da4 is 50, key is test_row_0/C:col10/1732666522854/Put/seqid=0 2024-11-27T00:15:23,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742075_1251 (size=12001) 2024-11-27T00:15:23,909 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5ad39b944d6e46efb240229fdcb98da4 2024-11-27T00:15:23,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/6eee32d620bf40c18538f420516d4bbf as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6eee32d620bf40c18538f420516d4bbf 2024-11-27T00:15:23,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6eee32d620bf40c18538f420516d4bbf, entries=150, sequenceid=33, filesize=11.7 K 2024-11-27T00:15:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/dfac0a77d38a4da89778d3ec1958a1d9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/dfac0a77d38a4da89778d3ec1958a1d9 2024-11-27T00:15:23,921 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/dfac0a77d38a4da89778d3ec1958a1d9, entries=150, sequenceid=33, filesize=11.7 K 2024-11-27T00:15:23,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5ad39b944d6e46efb240229fdcb98da4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5ad39b944d6e46efb240229fdcb98da4 2024-11-27T00:15:23,940 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5ad39b944d6e46efb240229fdcb98da4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-27T00:15:23,941 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 40e749124bfcc05793d8d1161b1bd9fa in 504ms, sequenceid=33, compaction requested=false 2024-11-27T00:15:23,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:23,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:23,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-27T00:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-27T00:15:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T00:15:23,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-27T00:15:23,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1260 sec 2024-11-27T00:15:23,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1380 sec 2024-11-27T00:15:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:24,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T00:15:24,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:24,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/fac0f2729e8a4883bd7249c7a5908d10 is 50, key is test_row_0/A:col10/1732666524015/Put/seqid=0 2024-11-27T00:15:24,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742076_1252 (size=11997) 2024-11-27T00:15:24,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/fac0f2729e8a4883bd7249c7a5908d10 2024-11-27T00:15:24,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/187aa9a7728842df8f5b5f67d16680d9 is 50, key is test_row_0/B:col10/1732666524015/Put/seqid=0 2024-11-27T00:15:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742077_1253 (size=9657) 2024-11-27T00:15:24,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/187aa9a7728842df8f5b5f67d16680d9 2024-11-27T00:15:24,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666584098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1d66f2af1ccc4bb1b17acf06a24bd7c0 is 50, key is test_row_0/C:col10/1732666524015/Put/seqid=0 2024-11-27T00:15:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742078_1254 (size=9657) 2024-11-27T00:15:24,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666584134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666584134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666584134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666584134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666584214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666584423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1d66f2af1ccc4bb1b17acf06a24bd7c0 2024-11-27T00:15:24,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/fac0f2729e8a4883bd7249c7a5908d10 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fac0f2729e8a4883bd7249c7a5908d10 2024-11-27T00:15:24,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fac0f2729e8a4883bd7249c7a5908d10, entries=150, sequenceid=46, filesize=11.7 K 2024-11-27T00:15:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/187aa9a7728842df8f5b5f67d16680d9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/187aa9a7728842df8f5b5f67d16680d9 2024-11-27T00:15:24,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/187aa9a7728842df8f5b5f67d16680d9, entries=100, sequenceid=46, filesize=9.4 K 2024-11-27T00:15:24,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1d66f2af1ccc4bb1b17acf06a24bd7c0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1d66f2af1ccc4bb1b17acf06a24bd7c0 2024-11-27T00:15:24,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1d66f2af1ccc4bb1b17acf06a24bd7c0, entries=100, sequenceid=46, filesize=9.4 K 2024-11-27T00:15:24,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 40e749124bfcc05793d8d1161b1bd9fa in 536ms, sequenceid=46, compaction requested=true 2024-11-27T00:15:24,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:24,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:24,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:24,553 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:24,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:24,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:24,553 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:24,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:24,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:24,554 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:24,554 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:24,554 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/B is initiating minor compaction (all files) 2024-11-27T00:15:24,554 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/A is initiating minor compaction (all files) 2024-11-27T00:15:24,554 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/B in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:24,554 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/cb915c7d49704cbb8dd379b065661910, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/dfac0a77d38a4da89778d3ec1958a1d9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/187aa9a7728842df8f5b5f67d16680d9] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=32.9 K 2024-11-27T00:15:24,554 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/A in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:24,554 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/ee04ba7bf43340c693f1269652dcd671, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6eee32d620bf40c18538f420516d4bbf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fac0f2729e8a4883bd7249c7a5908d10] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=35.2 K 2024-11-27T00:15:24,554 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting cb915c7d49704cbb8dd379b065661910, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732666522816 2024-11-27T00:15:24,554 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee04ba7bf43340c693f1269652dcd671, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732666522816 2024-11-27T00:15:24,555 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting dfac0a77d38a4da89778d3ec1958a1d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732666522854 2024-11-27T00:15:24,555 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eee32d620bf40c18538f420516d4bbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732666522854 2024-11-27T00:15:24,555 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 187aa9a7728842df8f5b5f67d16680d9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666523504 2024-11-27T00:15:24,555 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac0f2729e8a4883bd7249c7a5908d10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666523500 2024-11-27T00:15:24,562 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:24,562 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/4c07b57b7da140fc96b8ecfbfd6c930a is 50, key is test_row_0/B:col10/1732666524015/Put/seqid=0 2024-11-27T00:15:24,566 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:24,566 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/f770e9e33ffd4d4db2ea088208c01105 is 50, key is test_row_0/A:col10/1732666524015/Put/seqid=0 2024-11-27T00:15:24,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742079_1255 (size=12104) 2024-11-27T00:15:24,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742080_1256 (size=12104) 2024-11-27T00:15:24,585 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/f770e9e33ffd4d4db2ea088208c01105 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/f770e9e33ffd4d4db2ea088208c01105 2024-11-27T00:15:24,589 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/A of 40e749124bfcc05793d8d1161b1bd9fa into f770e9e33ffd4d4db2ea088208c01105(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:24,589 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:24,589 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/A, priority=13, startTime=1732666524553; duration=0sec 2024-11-27T00:15:24,589 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:24,589 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:A 2024-11-27T00:15:24,590 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:24,591 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:24,591 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/C is initiating minor compaction (all files) 2024-11-27T00:15:24,591 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/C in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:24,591 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/ee48bd9bbf604c1b84c12b45b95970c1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5ad39b944d6e46efb240229fdcb98da4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1d66f2af1ccc4bb1b17acf06a24bd7c0] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=32.9 K 2024-11-27T00:15:24,591 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee48bd9bbf604c1b84c12b45b95970c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732666522816 2024-11-27T00:15:24,592 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad39b944d6e46efb240229fdcb98da4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732666522854 2024-11-27T00:15:24,592 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d66f2af1ccc4bb1b17acf06a24bd7c0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666523504 2024-11-27T00:15:24,600 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:24,600 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/a775c9270c87419a8edae3c23165ac99 is 50, key is test_row_0/C:col10/1732666524015/Put/seqid=0 2024-11-27T00:15:24,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742081_1257 (size=12104) 2024-11-27T00:15:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:24,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:15:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:24,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:24,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/60bbdb06891a4f1998bbf7dde57d689e is 50, key is test_row_0/A:col10/1732666524661/Put/seqid=0 2024-11-27T00:15:24,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742082_1258 (size=12001) 2024-11-27T00:15:24,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/60bbdb06891a4f1998bbf7dde57d689e 2024-11-27T00:15:24,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666584670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666584672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666584672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666584674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/63c9b958fcb14528bdac7236224b58f5 is 50, key is test_row_0/B:col10/1732666524661/Put/seqid=0 2024-11-27T00:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742083_1259 (size=12001) 2024-11-27T00:15:24,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666584733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666584784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666584784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666584784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666584784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T00:15:24,964 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:24,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-27T00:15:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T00:15:24,966 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:24,966 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:24,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:24,975 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/4c07b57b7da140fc96b8ecfbfd6c930a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4c07b57b7da140fc96b8ecfbfd6c930a 2024-11-27T00:15:24,982 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/B of 40e749124bfcc05793d8d1161b1bd9fa into 4c07b57b7da140fc96b8ecfbfd6c930a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:24,982 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:24,982 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/B, priority=13, startTime=1732666524553; duration=0sec 2024-11-27T00:15:24,982 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:24,982 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:B 2024-11-27T00:15:24,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666584994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666584994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666584994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666584994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,019 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/a775c9270c87419a8edae3c23165ac99 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a775c9270c87419a8edae3c23165ac99 2024-11-27T00:15:25,022 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/C of 40e749124bfcc05793d8d1161b1bd9fa into a775c9270c87419a8edae3c23165ac99(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:25,022 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:25,023 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/C, priority=13, startTime=1732666524553; duration=0sec 2024-11-27T00:15:25,023 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:25,023 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:C 2024-11-27T00:15:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T00:15:25,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/63c9b958fcb14528bdac7236224b58f5 2024-11-27T00:15:25,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1105579bb2734c62a892a69056492a9a is 50, key is test_row_0/C:col10/1732666524661/Put/seqid=0 2024-11-27T00:15:25,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742084_1260 (size=12001) 2024-11-27T00:15:25,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T00:15:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:25,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666585244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T00:15:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:25,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T00:15:25,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666585303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666585304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666585304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666585304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T00:15:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:25,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:25,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1105579bb2734c62a892a69056492a9a 2024-11-27T00:15:25,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/60bbdb06891a4f1998bbf7dde57d689e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/60bbdb06891a4f1998bbf7dde57d689e 2024-11-27T00:15:25,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/60bbdb06891a4f1998bbf7dde57d689e, entries=150, sequenceid=68, filesize=11.7 K 2024-11-27T00:15:25,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/63c9b958fcb14528bdac7236224b58f5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/63c9b958fcb14528bdac7236224b58f5 2024-11-27T00:15:25,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/63c9b958fcb14528bdac7236224b58f5, entries=150, sequenceid=68, filesize=11.7 K 2024-11-27T00:15:25,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1105579bb2734c62a892a69056492a9a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1105579bb2734c62a892a69056492a9a 2024-11-27T00:15:25,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1105579bb2734c62a892a69056492a9a, entries=150, sequenceid=68, filesize=11.7 K 2024-11-27T00:15:25,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 40e749124bfcc05793d8d1161b1bd9fa in 870ms, sequenceid=68, compaction requested=false 2024-11-27T00:15:25,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:25,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T00:15:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:25,579 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T00:15:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:25,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/643ae48a15c142159c1e1526da52d547 is 50, key is test_row_0/A:col10/1732666524673/Put/seqid=0 2024-11-27T00:15:25,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742085_1261 (size=12001) 2024-11-27T00:15:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T00:15:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:25,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:25,830 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T00:15:25,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666585833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666525825, max timestamp=1732666525837Num uniques 859; 2024-11-27T00:15:25,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666525825, max timestamp=1732666525837Num uniques 859; 2024-11-27T00:15:25,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:25,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:25,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666525825, max timestamp=1732666525837Num uniques 859; 2024-11-27T00:15:25,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:25,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666585863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666585868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666585869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666585944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666585974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,987 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/643ae48a15c142159c1e1526da52d547 2024-11-27T00:15:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666585983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666585983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:25,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/78e027de517b4b57bcc72f8238df91f0 is 50, key is test_row_0/B:col10/1732666524673/Put/seqid=0 2024-11-27T00:15:25,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742086_1262 (size=12001) 2024-11-27T00:15:25,999 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/78e027de517b4b57bcc72f8238df91f0 2024-11-27T00:15:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/81dbb1e50a924a888e683ad02aae9d1c is 50, key is test_row_0/C:col10/1732666524673/Put/seqid=0 2024-11-27T00:15:26,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742087_1263 (size=12001) 2024-11-27T00:15:26,010 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/81dbb1e50a924a888e683ad02aae9d1c 2024-11-27T00:15:26,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/643ae48a15c142159c1e1526da52d547 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/643ae48a15c142159c1e1526da52d547 2024-11-27T00:15:26,017 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/643ae48a15c142159c1e1526da52d547, entries=150, sequenceid=81, filesize=11.7 K 2024-11-27T00:15:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/78e027de517b4b57bcc72f8238df91f0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/78e027de517b4b57bcc72f8238df91f0 2024-11-27T00:15:26,021 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/78e027de517b4b57bcc72f8238df91f0, entries=150, sequenceid=81, filesize=11.7 K 2024-11-27T00:15:26,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/81dbb1e50a924a888e683ad02aae9d1c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/81dbb1e50a924a888e683ad02aae9d1c 2024-11-27T00:15:26,026 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/81dbb1e50a924a888e683ad02aae9d1c, entries=150, sequenceid=81, filesize=11.7 K 2024-11-27T00:15:26,027 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 40e749124bfcc05793d8d1161b1bd9fa in 449ms, sequenceid=81, compaction requested=true 2024-11-27T00:15:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-27T00:15:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-27T00:15:26,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-27T00:15:26,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0610 sec 2024-11-27T00:15:26,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0660 sec 2024-11-27T00:15:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T00:15:26,104 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:26,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-27T00:15:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T00:15:26,105 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:26,106 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:26,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:26,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T00:15:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:26,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/564b86bafed5402790f51ed42df223a5 is 50, key is test_row_0/A:col10/1732666526156/Put/seqid=0 2024-11-27T00:15:26,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742088_1264 (size=14341) 2024-11-27T00:15:26,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666586188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666586190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666586194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666586194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T00:15:26,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666586253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,255 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666586304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666586304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T00:15:26,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666586504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666586504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666586514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666586514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/564b86bafed5402790f51ed42df223a5 2024-11-27T00:15:26,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/864d05f5af7141109eda1734ba0e2f72 is 50, key is test_row_0/B:col10/1732666526156/Put/seqid=0 2024-11-27T00:15:26,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742089_1265 (size=12001) 2024-11-27T00:15:26,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T00:15:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666586824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666586824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:26,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:26,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:26,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/864d05f5af7141109eda1734ba0e2f72 2024-11-27T00:15:26,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/140eb61cdd484d7f90b84398a0d1a993 is 50, key is test_row_0/C:col10/1732666526156/Put/seqid=0 2024-11-27T00:15:26,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742090_1266 (size=12001) 2024-11-27T00:15:27,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666587014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:27,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666587014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:27,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:27,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:27,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:27,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T00:15:27,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:27,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666587334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:27,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666587334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:27,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/140eb61cdd484d7f90b84398a0d1a993 2024-11-27T00:15:27,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/564b86bafed5402790f51ed42df223a5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/564b86bafed5402790f51ed42df223a5 2024-11-27T00:15:27,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/564b86bafed5402790f51ed42df223a5, entries=200, sequenceid=117, filesize=14.0 K 2024-11-27T00:15:27,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/864d05f5af7141109eda1734ba0e2f72 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/864d05f5af7141109eda1734ba0e2f72 2024-11-27T00:15:27,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/864d05f5af7141109eda1734ba0e2f72, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T00:15:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/140eb61cdd484d7f90b84398a0d1a993 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/140eb61cdd484d7f90b84398a0d1a993 2024-11-27T00:15:27,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/140eb61cdd484d7f90b84398a0d1a993, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T00:15:27,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 40e749124bfcc05793d8d1161b1bd9fa in 1250ms, sequenceid=117, compaction requested=true 2024-11-27T00:15:27,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:27,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:27,408 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:27,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:27,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:27,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:27,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:27,408 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:27,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:27,409 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:27,409 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:27,409 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/B is initiating minor compaction (all files) 2024-11-27T00:15:27,409 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/A is initiating minor compaction (all files) 2024-11-27T00:15:27,409 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/A in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,409 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/B in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,409 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/f770e9e33ffd4d4db2ea088208c01105, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/60bbdb06891a4f1998bbf7dde57d689e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/643ae48a15c142159c1e1526da52d547, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/564b86bafed5402790f51ed42df223a5] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=49.3 K 2024-11-27T00:15:27,409 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4c07b57b7da140fc96b8ecfbfd6c930a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/63c9b958fcb14528bdac7236224b58f5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/78e027de517b4b57bcc72f8238df91f0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/864d05f5af7141109eda1734ba0e2f72] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=47.0 K 2024-11-27T00:15:27,410 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f770e9e33ffd4d4db2ea088208c01105, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666522854 2024-11-27T00:15:27,410 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c07b57b7da140fc96b8ecfbfd6c930a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666522854 2024-11-27T00:15:27,410 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60bbdb06891a4f1998bbf7dde57d689e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732666524090 2024-11-27T00:15:27,410 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 63c9b958fcb14528bdac7236224b58f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732666524090 2024-11-27T00:15:27,410 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 643ae48a15c142159c1e1526da52d547, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732666524669 2024-11-27T00:15:27,411 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e027de517b4b57bcc72f8238df91f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732666524669 2024-11-27T00:15:27,411 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 564b86bafed5402790f51ed42df223a5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732666525861 2024-11-27T00:15:27,411 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 864d05f5af7141109eda1734ba0e2f72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732666525867 2024-11-27T00:15:27,419 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:27,419 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/3d955d029b69466b98d7113a5f84afe5 is 50, key is test_row_0/A:col10/1732666526156/Put/seqid=0 2024-11-27T00:15:27,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742091_1267 (size=12241) 2024-11-27T00:15:27,426 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:27,426 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/ed2ae9562abd40db83a9c4a716301704 is 50, key is test_row_0/B:col10/1732666526156/Put/seqid=0 2024-11-27T00:15:27,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742092_1268 (size=12241) 2024-11-27T00:15:27,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T00:15:27,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,485 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:27,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4db05d6b41574eecba060afd0a9d9052 is 50, key is test_row_0/A:col10/1732666526187/Put/seqid=0 2024-11-27T00:15:27,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742093_1269 (size=12001) 2024-11-27T00:15:27,830 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/3d955d029b69466b98d7113a5f84afe5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3d955d029b69466b98d7113a5f84afe5 2024-11-27T00:15:27,836 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/A of 40e749124bfcc05793d8d1161b1bd9fa into 3d955d029b69466b98d7113a5f84afe5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:27,836 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:27,836 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/A, priority=12, startTime=1732666527408; duration=0sec 2024-11-27T00:15:27,836 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/ed2ae9562abd40db83a9c4a716301704 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ed2ae9562abd40db83a9c4a716301704 2024-11-27T00:15:27,836 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:27,836 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:A 2024-11-27T00:15:27,836 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:15:27,838 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:15:27,838 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/C is initiating minor compaction (all files) 2024-11-27T00:15:27,838 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/C in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:27,838 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a775c9270c87419a8edae3c23165ac99, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1105579bb2734c62a892a69056492a9a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/81dbb1e50a924a888e683ad02aae9d1c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/140eb61cdd484d7f90b84398a0d1a993] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=47.0 K 2024-11-27T00:15:27,840 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a775c9270c87419a8edae3c23165ac99, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732666522854 2024-11-27T00:15:27,840 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1105579bb2734c62a892a69056492a9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732666524090 2024-11-27T00:15:27,840 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81dbb1e50a924a888e683ad02aae9d1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732666524669 2024-11-27T00:15:27,841 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 140eb61cdd484d7f90b84398a0d1a993, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732666525867 2024-11-27T00:15:27,843 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/B of 40e749124bfcc05793d8d1161b1bd9fa into ed2ae9562abd40db83a9c4a716301704(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:27,843 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:27,843 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/B, priority=12, startTime=1732666527408; duration=0sec 2024-11-27T00:15:27,843 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:27,843 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:B 2024-11-27T00:15:27,848 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:27,849 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/6e247d54823345c184af508902c60e7e is 50, key is test_row_0/C:col10/1732666526156/Put/seqid=0 2024-11-27T00:15:27,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742094_1270 (size=12241) 2024-11-27T00:15:27,870 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/6e247d54823345c184af508902c60e7e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/6e247d54823345c184af508902c60e7e 2024-11-27T00:15:27,875 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/C of 40e749124bfcc05793d8d1161b1bd9fa into 6e247d54823345c184af508902c60e7e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:27,875 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:27,875 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/C, priority=12, startTime=1732666527408; duration=0sec 2024-11-27T00:15:27,875 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:27,875 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:C 2024-11-27T00:15:27,894 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4db05d6b41574eecba060afd0a9d9052 2024-11-27T00:15:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5427558bec4d418dbff50a41022b4579 is 50, key is test_row_0/B:col10/1732666526187/Put/seqid=0 2024-11-27T00:15:27,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742095_1271 (size=12001) 2024-11-27T00:15:27,904 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5427558bec4d418dbff50a41022b4579 2024-11-27T00:15:27,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/3fd7144dbc744e8f86bb0cacf9f7543b is 50, key is test_row_0/C:col10/1732666526187/Put/seqid=0 2024-11-27T00:15:27,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742096_1272 (size=12001) 2024-11-27T00:15:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:28,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666528046, max timestamp=1732666528057Num uniques 859; 2024-11-27T00:15:28,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666528046, max timestamp=1732666528057Num uniques 859; 2024-11-27T00:15:28,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666528046, max timestamp=1732666528057Num uniques 859; 2024-11-27T00:15:28,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:15:28,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:15:28,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:15:28,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666588102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666588107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666588214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666588214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T00:15:28,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666588263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,268 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,313 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/3fd7144dbc744e8f86bb0cacf9f7543b 2024-11-27T00:15:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4db05d6b41574eecba060afd0a9d9052 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4db05d6b41574eecba060afd0a9d9052 2024-11-27T00:15:28,322 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4db05d6b41574eecba060afd0a9d9052, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T00:15:28,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5427558bec4d418dbff50a41022b4579 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5427558bec4d418dbff50a41022b4579 2024-11-27T00:15:28,328 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5427558bec4d418dbff50a41022b4579, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T00:15:28,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/3fd7144dbc744e8f86bb0cacf9f7543b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3fd7144dbc744e8f86bb0cacf9f7543b 2024-11-27T00:15:28,332 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3fd7144dbc744e8f86bb0cacf9f7543b, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T00:15:28,333 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 40e749124bfcc05793d8d1161b1bd9fa in 848ms, sequenceid=127, compaction requested=false 2024-11-27T00:15:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-27T00:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-27T00:15:28,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-27T00:15:28,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2290 sec 2024-11-27T00:15:28,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.2330 sec 2024-11-27T00:15:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:28,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-27T00:15:28,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:28,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T00:15:28,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:28,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T00:15:28,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:28,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T00:15:28,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/248953ac0287442abc40dce3a16eb240 is 50, key is test_row_0/A:col10/1732666528086/Put/seqid=0 2024-11-27T00:15:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742097_1273 (size=14541) 2024-11-27T00:15:28,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666588365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666588369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666588424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666588424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666588474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666588474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666588684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666588684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666588734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666588734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/248953ac0287442abc40dce3a16eb240 2024-11-27T00:15:28,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/29236f7c15364fc2a67a57b7eb22603d is 50, key is test_row_0/B:col10/1732666528086/Put/seqid=0 2024-11-27T00:15:28,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742098_1274 (size=12151) 2024-11-27T00:15:28,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/29236f7c15364fc2a67a57b7eb22603d 2024-11-27T00:15:28,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5b492e4b398d45ac8f51d64d5fd3af0a is 50, key is test_row_0/C:col10/1732666528086/Put/seqid=0 2024-11-27T00:15:28,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742099_1275 (size=12151) 2024-11-27T00:15:28,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666588994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666588994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5b492e4b398d45ac8f51d64d5fd3af0a 2024-11-27T00:15:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/248953ac0287442abc40dce3a16eb240 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/248953ac0287442abc40dce3a16eb240 2024-11-27T00:15:29,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/248953ac0287442abc40dce3a16eb240, entries=200, sequenceid=168, filesize=14.2 K 2024-11-27T00:15:29,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/29236f7c15364fc2a67a57b7eb22603d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/29236f7c15364fc2a67a57b7eb22603d 2024-11-27T00:15:29,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/29236f7c15364fc2a67a57b7eb22603d, entries=150, sequenceid=168, filesize=11.9 K 2024-11-27T00:15:29,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5b492e4b398d45ac8f51d64d5fd3af0a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5b492e4b398d45ac8f51d64d5fd3af0a 2024-11-27T00:15:29,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5b492e4b398d45ac8f51d64d5fd3af0a, entries=150, sequenceid=168, filesize=11.9 K 2024-11-27T00:15:29,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 40e749124bfcc05793d8d1161b1bd9fa in 856ms, sequenceid=168, compaction requested=true 2024-11-27T00:15:29,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:29,205 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:29,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:29,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:29,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:29,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:29,205 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:29,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:29,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:29,205 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:29,205 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:29,205 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/B is initiating minor compaction (all files) 2024-11-27T00:15:29,205 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/A is initiating minor compaction (all files) 2024-11-27T00:15:29,206 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/B in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:29,206 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/A in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:29,206 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3d955d029b69466b98d7113a5f84afe5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4db05d6b41574eecba060afd0a9d9052, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/248953ac0287442abc40dce3a16eb240] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=37.9 K 2024-11-27T00:15:29,206 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ed2ae9562abd40db83a9c4a716301704, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5427558bec4d418dbff50a41022b4579, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/29236f7c15364fc2a67a57b7eb22603d] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=35.5 K 2024-11-27T00:15:29,206 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ed2ae9562abd40db83a9c4a716301704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732666525867 2024-11-27T00:15:29,206 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d955d029b69466b98d7113a5f84afe5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732666525867 2024-11-27T00:15:29,206 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5427558bec4d418dbff50a41022b4579, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732666526182 2024-11-27T00:15:29,207 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 29236f7c15364fc2a67a57b7eb22603d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732666528086 2024-11-27T00:15:29,207 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4db05d6b41574eecba060afd0a9d9052, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732666526182 2024-11-27T00:15:29,207 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 248953ac0287442abc40dce3a16eb240, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732666528086 2024-11-27T00:15:29,217 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:29,218 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/352a245b38c64c9c8463b9aa36dcec93 is 50, key is test_row_0/B:col10/1732666528086/Put/seqid=0 2024-11-27T00:15:29,226 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:29,227 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/7819879e20eb47cca185246f2329a0f5 is 50, key is test_row_0/A:col10/1732666528086/Put/seqid=0 2024-11-27T00:15:29,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742100_1276 (size=12493) 2024-11-27T00:15:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742101_1277 (size=12493) 2024-11-27T00:15:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:29,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:29,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:29,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/b69360f6126e443788b98d321e705f69 is 50, key is test_row_0/A:col10/1732666529244/Put/seqid=0 2024-11-27T00:15:29,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742102_1278 (size=9757) 2024-11-27T00:15:29,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666529272, max timestamp=1732666529287Num uniques 859; 2024-11-27T00:15:29,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666529272, max timestamp=1732666529287Num uniques 859; 2024-11-27T00:15:29,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:29,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666529272, max timestamp=1732666529287Num uniques 859; 2024-11-27T00:15:29,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:29,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:29,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666589325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666589331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666589434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666589444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666589504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666589504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,635 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/352a245b38c64c9c8463b9aa36dcec93 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/352a245b38c64c9c8463b9aa36dcec93 2024-11-27T00:15:29,638 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/7819879e20eb47cca185246f2329a0f5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/7819879e20eb47cca185246f2329a0f5 2024-11-27T00:15:29,639 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/B of 40e749124bfcc05793d8d1161b1bd9fa into 352a245b38c64c9c8463b9aa36dcec93(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:29,639 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:29,639 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/B, priority=13, startTime=1732666529205; duration=0sec 2024-11-27T00:15:29,639 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:29,639 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:B 2024-11-27T00:15:29,639 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:29,640 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:29,640 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/C is initiating minor compaction (all files) 2024-11-27T00:15:29,640 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/C in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:29,640 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/6e247d54823345c184af508902c60e7e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3fd7144dbc744e8f86bb0cacf9f7543b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5b492e4b398d45ac8f51d64d5fd3af0a] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=35.5 K 2024-11-27T00:15:29,641 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e247d54823345c184af508902c60e7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732666525867 2024-11-27T00:15:29,641 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd7144dbc744e8f86bb0cacf9f7543b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732666526182 2024-11-27T00:15:29,641 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b492e4b398d45ac8f51d64d5fd3af0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732666528086 2024-11-27T00:15:29,645 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/A of 40e749124bfcc05793d8d1161b1bd9fa into 7819879e20eb47cca185246f2329a0f5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:29,645 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:29,645 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/A, priority=13, startTime=1732666529205; duration=0sec 2024-11-27T00:15:29,645 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:29,645 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:A 2024-11-27T00:15:29,649 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:29,649 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/b5b0e753016e46849f9c9a950f6a48fc is 50, key is test_row_0/C:col10/1732666528086/Put/seqid=0 2024-11-27T00:15:29,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666589644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/b69360f6126e443788b98d321e705f69 2024-11-27T00:15:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666589653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/584cb0af91ef4f39b92400b6a2b50bdf is 50, key is test_row_0/B:col10/1732666529244/Put/seqid=0 2024-11-27T00:15:29,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742103_1279 (size=12493) 2024-11-27T00:15:29,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742104_1280 (size=9757) 2024-11-27T00:15:29,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/584cb0af91ef4f39b92400b6a2b50bdf 2024-11-27T00:15:29,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/a40d9e752934472da03f69d753a7d7d8 is 50, key is test_row_0/C:col10/1732666529244/Put/seqid=0 2024-11-27T00:15:29,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742105_1281 (size=9757) 2024-11-27T00:15:29,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666589954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:29,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666589964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,081 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/b5b0e753016e46849f9c9a950f6a48fc as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b5b0e753016e46849f9c9a950f6a48fc 2024-11-27T00:15:30,086 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/C of 40e749124bfcc05793d8d1161b1bd9fa into b5b0e753016e46849f9c9a950f6a48fc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:30,086 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:30,086 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/C, priority=13, startTime=1732666529205; duration=0sec 2024-11-27T00:15:30,086 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:30,086 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:C 2024-11-27T00:15:30,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/a40d9e752934472da03f69d753a7d7d8 2024-11-27T00:15:30,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/b69360f6126e443788b98d321e705f69 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b69360f6126e443788b98d321e705f69 2024-11-27T00:15:30,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b69360f6126e443788b98d321e705f69, entries=100, sequenceid=178, filesize=9.5 K 2024-11-27T00:15:30,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/584cb0af91ef4f39b92400b6a2b50bdf as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/584cb0af91ef4f39b92400b6a2b50bdf 2024-11-27T00:15:30,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/584cb0af91ef4f39b92400b6a2b50bdf, entries=100, sequenceid=178, filesize=9.5 K 2024-11-27T00:15:30,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/a40d9e752934472da03f69d753a7d7d8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a40d9e752934472da03f69d753a7d7d8 2024-11-27T00:15:30,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a40d9e752934472da03f69d753a7d7d8, entries=100, sequenceid=178, filesize=9.5 K 2024-11-27T00:15:30,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 40e749124bfcc05793d8d1161b1bd9fa in 876ms, sequenceid=178, compaction requested=false 2024-11-27T00:15:30,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T00:15:30,274 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:30,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-27T00:15:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T00:15:30,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:30,277 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:30,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T00:15:30,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T00:15:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:30,432 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T00:15:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:30,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:30,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/bc6b386dcec442f19fb365a0846432af is 50, key is test_row_0/A:col10/1732666529317/Put/seqid=0 2024-11-27T00:15:30,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742106_1282 (size=12151) 2024-11-27T00:15:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:30,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:30,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666590484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666590486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666590514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:30,519 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666590524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:30,528 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T00:15:30,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666590594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666590594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:30,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666590804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666590804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:30,843 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/bc6b386dcec442f19fb365a0846432af 2024-11-27T00:15:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/6d668bd9ef774918b58d64398c6f4ffb is 50, key is test_row_0/B:col10/1732666529317/Put/seqid=0 2024-11-27T00:15:30,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742107_1283 (size=12151) 2024-11-27T00:15:30,855 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/6d668bd9ef774918b58d64398c6f4ffb 2024-11-27T00:15:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5603293d8abe4d1aa6cbaba48d8c4ac9 is 50, key is test_row_0/C:col10/1732666529317/Put/seqid=0 2024-11-27T00:15:30,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742108_1284 (size=12151) 2024-11-27T00:15:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T00:15:31,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666591113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:31,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666591114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:31,265 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5603293d8abe4d1aa6cbaba48d8c4ac9 2024-11-27T00:15:31,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/bc6b386dcec442f19fb365a0846432af as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/bc6b386dcec442f19fb365a0846432af 2024-11-27T00:15:31,273 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/bc6b386dcec442f19fb365a0846432af, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T00:15:31,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/6d668bd9ef774918b58d64398c6f4ffb as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/6d668bd9ef774918b58d64398c6f4ffb 2024-11-27T00:15:31,277 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/6d668bd9ef774918b58d64398c6f4ffb, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T00:15:31,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/5603293d8abe4d1aa6cbaba48d8c4ac9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5603293d8abe4d1aa6cbaba48d8c4ac9 2024-11-27T00:15:31,281 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5603293d8abe4d1aa6cbaba48d8c4ac9, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T00:15:31,281 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 40e749124bfcc05793d8d1161b1bd9fa in 849ms, sequenceid=217, compaction requested=true 2024-11-27T00:15:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-27T00:15:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-27T00:15:31,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-27T00:15:31,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-11-27T00:15:31,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0100 sec 2024-11-27T00:15:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T00:15:31,414 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:31,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-27T00:15:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T00:15:31,416 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:31,417 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:31,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T00:15:31,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T00:15:31,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:31,569 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:31,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:31,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/832ad8fa87ac435b8f09fa8ff14b94da is 50, key is test_row_0/A:col10/1732666530482/Put/seqid=0 2024-11-27T00:15:31,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742109_1285 (size=12151) 2024-11-27T00:15:31,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666531654, max timestamp=1732666531663Num uniques 859; 2024-11-27T00:15:31,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666531654, max timestamp=1732666531663Num uniques 859; 2024-11-27T00:15:31,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:15:31,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:15:31,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666531654, max timestamp=1732666531663Num uniques 859; 2024-11-27T00:15:31,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:15:31,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666591714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:31,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666591715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T00:15:31,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666591824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:31,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666591824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:31,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666592034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666592034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,038 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/832ad8fa87ac435b8f09fa8ff14b94da 2024-11-27T00:15:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T00:15:32,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5a1eec7168d04e80ab41e33b0c4a0f6b is 50, key is test_row_0/B:col10/1732666530482/Put/seqid=0 2024-11-27T00:15:32,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742110_1286 (size=12151) 2024-11-27T00:15:32,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666592284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,286 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666592343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666592343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,449 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5a1eec7168d04e80ab41e33b0c4a0f6b 2024-11-27T00:15:32,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/58f32602525d4561b53bc96da2d98461 is 50, key is test_row_0/C:col10/1732666530482/Put/seqid=0 2024-11-27T00:15:32,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742111_1287 (size=12151) 2024-11-27T00:15:32,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666592544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666592544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,547 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,547 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T00:15:32,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666592853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666592853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:32,859 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/58f32602525d4561b53bc96da2d98461 2024-11-27T00:15:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/832ad8fa87ac435b8f09fa8ff14b94da as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/832ad8fa87ac435b8f09fa8ff14b94da 2024-11-27T00:15:32,867 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/832ad8fa87ac435b8f09fa8ff14b94da, entries=150, sequenceid=227, filesize=11.9 K 2024-11-27T00:15:32,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5a1eec7168d04e80ab41e33b0c4a0f6b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5a1eec7168d04e80ab41e33b0c4a0f6b 2024-11-27T00:15:32,872 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5a1eec7168d04e80ab41e33b0c4a0f6b, entries=150, sequenceid=227, filesize=11.9 K 2024-11-27T00:15:32,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/58f32602525d4561b53bc96da2d98461 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/58f32602525d4561b53bc96da2d98461 2024-11-27T00:15:32,879 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/58f32602525d4561b53bc96da2d98461, entries=150, sequenceid=227, filesize=11.9 K 2024-11-27T00:15:32,880 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 40e749124bfcc05793d8d1161b1bd9fa in 1310ms, sequenceid=227, compaction requested=true 2024-11-27T00:15:32,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:32,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:32,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-27T00:15:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-27T00:15:32,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-27T00:15:32,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4640 sec 2024-11-27T00:15:32,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4680 sec 2024-11-27T00:15:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T00:15:33,564 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:33,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-27T00:15:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T00:15:33,566 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:33,567 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:33,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T00:15:33,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T00:15:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:33,720 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-27T00:15:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:33,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/3093e375875c4e9fa2e07e2dd9bca6b4 is 50, key is test_row_0/A:col10/1732666531713/Put/seqid=0 2024-11-27T00:15:33,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742112_1288 (size=12301) 2024-11-27T00:15:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:33,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T00:15:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666593892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:33,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666593894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666594003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:34,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666594004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:34,128 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/3093e375875c4e9fa2e07e2dd9bca6b4 2024-11-27T00:15:34,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5ded6f1e54bb4eb89e703966c477f2ff is 50, key is test_row_0/B:col10/1732666531713/Put/seqid=0 2024-11-27T00:15:34,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742113_1289 (size=12301) 2024-11-27T00:15:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T00:15:34,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666594213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:34,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666594213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:34,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666594524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:34,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666594524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:34,539 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5ded6f1e54bb4eb89e703966c477f2ff 2024-11-27T00:15:34,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1941057103a446a4b424097b09931241 is 50, key is test_row_0/C:col10/1732666531713/Put/seqid=0 2024-11-27T00:15:34,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742114_1290 (size=12301) 2024-11-27T00:15:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T00:15:34,952 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1941057103a446a4b424097b09931241 2024-11-27T00:15:34,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/3093e375875c4e9fa2e07e2dd9bca6b4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3093e375875c4e9fa2e07e2dd9bca6b4 2024-11-27T00:15:34,960 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3093e375875c4e9fa2e07e2dd9bca6b4, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T00:15:34,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/5ded6f1e54bb4eb89e703966c477f2ff as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5ded6f1e54bb4eb89e703966c477f2ff 2024-11-27T00:15:34,964 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5ded6f1e54bb4eb89e703966c477f2ff, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T00:15:34,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1941057103a446a4b424097b09931241 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1941057103a446a4b424097b09931241 2024-11-27T00:15:34,968 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1941057103a446a4b424097b09931241, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T00:15:34,969 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 40e749124bfcc05793d8d1161b1bd9fa in 1249ms, sequenceid=264, compaction requested=true 2024-11-27T00:15:34,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:34,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:34,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-27T00:15:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-27T00:15:34,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-27T00:15:34,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-27T00:15:34,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4070 sec 2024-11-27T00:15:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:35,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T00:15:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:35,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/790fa936687540ceb9783f8541597250 is 50, key is test_row_0/A:col10/1732666535036/Put/seqid=0 2024-11-27T00:15:35,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742115_1291 (size=17181) 2024-11-27T00:15:35,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666595087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666595093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:35,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666595204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:35,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666595204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:35,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666595414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666595414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:35,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/790fa936687540ceb9783f8541597250 2024-11-27T00:15:35,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/4313223a0b6446df9ad04c9725235ce3 is 50, key is test_row_0/B:col10/1732666535036/Put/seqid=0 2024-11-27T00:15:35,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742116_1292 (size=12301) 2024-11-27T00:15:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T00:15:35,714 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:35,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-27T00:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T00:15:35,716 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:35,717 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:35,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666595724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666595724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T00:15:35,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/4313223a0b6446df9ad04c9725235ce3 2024-11-27T00:15:35,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/b2e869aa8be743f3afcf04a47a81ca31 is 50, key is test_row_0/C:col10/1732666535036/Put/seqid=0 2024-11-27T00:15:35,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742117_1293 (size=12301) 2024-11-27T00:15:35,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T00:15:35,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:35,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:35,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:35,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:35,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:36,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T00:15:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:36,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:36,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T00:15:36,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T00:15:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:36,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:36,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:36,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666596234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:36,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666596234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:36,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/b2e869aa8be743f3afcf04a47a81ca31 2024-11-27T00:15:36,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/790fa936687540ceb9783f8541597250 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/790fa936687540ceb9783f8541597250 2024-11-27T00:15:36,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/790fa936687540ceb9783f8541597250, entries=250, sequenceid=276, filesize=16.8 K 2024-11-27T00:15:36,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/4313223a0b6446df9ad04c9725235ce3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4313223a0b6446df9ad04c9725235ce3 2024-11-27T00:15:36,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4313223a0b6446df9ad04c9725235ce3, entries=150, sequenceid=276, filesize=12.0 K 2024-11-27T00:15:36,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/b2e869aa8be743f3afcf04a47a81ca31 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b2e869aa8be743f3afcf04a47a81ca31 2024-11-27T00:15:36,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b2e869aa8be743f3afcf04a47a81ca31, entries=150, sequenceid=276, filesize=12.0 K 2024-11-27T00:15:36,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 40e749124bfcc05793d8d1161b1bd9fa in 1248ms, sequenceid=276, compaction requested=true 2024-11-27T00:15:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:36,285 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T00:15:36,285 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T00:15:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:36,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:36,286 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 76034 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T00:15:36,286 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 71154 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T00:15:36,286 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/A is initiating minor compaction (all files) 2024-11-27T00:15:36,286 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/B is initiating minor compaction (all files) 2024-11-27T00:15:36,286 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/A in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:36,286 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/B in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:36,287 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/7819879e20eb47cca185246f2329a0f5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b69360f6126e443788b98d321e705f69, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/bc6b386dcec442f19fb365a0846432af, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/832ad8fa87ac435b8f09fa8ff14b94da, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3093e375875c4e9fa2e07e2dd9bca6b4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/790fa936687540ceb9783f8541597250] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=74.3 K 2024-11-27T00:15:36,287 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/352a245b38c64c9c8463b9aa36dcec93, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/584cb0af91ef4f39b92400b6a2b50bdf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/6d668bd9ef774918b58d64398c6f4ffb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5a1eec7168d04e80ab41e33b0c4a0f6b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5ded6f1e54bb4eb89e703966c477f2ff, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4313223a0b6446df9ad04c9725235ce3] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=69.5 K 2024-11-27T00:15:36,287 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7819879e20eb47cca185246f2329a0f5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732666528086 2024-11-27T00:15:36,287 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 352a245b38c64c9c8463b9aa36dcec93, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732666528086 2024-11-27T00:15:36,287 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting b69360f6126e443788b98d321e705f69, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732666528363 2024-11-27T00:15:36,287 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 584cb0af91ef4f39b92400b6a2b50bdf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732666528363 2024-11-27T00:15:36,287 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc6b386dcec442f19fb365a0846432af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732666529317 2024-11-27T00:15:36,288 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d668bd9ef774918b58d64398c6f4ffb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732666529317 2024-11-27T00:15:36,288 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 832ad8fa87ac435b8f09fa8ff14b94da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732666530480 2024-11-27T00:15:36,288 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a1eec7168d04e80ab41e33b0c4a0f6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732666530480 2024-11-27T00:15:36,288 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3093e375875c4e9fa2e07e2dd9bca6b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732666531705 2024-11-27T00:15:36,288 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ded6f1e54bb4eb89e703966c477f2ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732666531705 2024-11-27T00:15:36,288 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 790fa936687540ceb9783f8541597250, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732666533882 2024-11-27T00:15:36,289 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4313223a0b6446df9ad04c9725235ce3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732666533892 2024-11-27T00:15:36,303 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#B#compaction#252 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:36,303 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/41e69e6289b94de89519c2aa43487da5 is 50, key is test_row_0/B:col10/1732666535036/Put/seqid=0 2024-11-27T00:15:36,305 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#A#compaction#253 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:36,305 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/b0b619b49fe84077b9c5da7b09854db1 is 50, key is test_row_0/A:col10/1732666535036/Put/seqid=0 2024-11-27T00:15:36,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742118_1294 (size=12847) 2024-11-27T00:15:36,314 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/41e69e6289b94de89519c2aa43487da5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/41e69e6289b94de89519c2aa43487da5 2024-11-27T00:15:36,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742119_1295 (size=12847) 2024-11-27T00:15:36,321 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/B of 40e749124bfcc05793d8d1161b1bd9fa into 41e69e6289b94de89519c2aa43487da5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:36,321 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:36,321 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/B, priority=10, startTime=1732666536285; duration=0sec 2024-11-27T00:15:36,321 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:36,321 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:B 2024-11-27T00:15:36,321 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T00:15:36,322 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 71154 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T00:15:36,322 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/C is initiating minor compaction (all files) 2024-11-27T00:15:36,323 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/C in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:36,323 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b5b0e753016e46849f9c9a950f6a48fc, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a40d9e752934472da03f69d753a7d7d8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5603293d8abe4d1aa6cbaba48d8c4ac9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/58f32602525d4561b53bc96da2d98461, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1941057103a446a4b424097b09931241, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b2e869aa8be743f3afcf04a47a81ca31] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=69.5 K 2024-11-27T00:15:36,323 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b0e753016e46849f9c9a950f6a48fc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732666528086 2024-11-27T00:15:36,323 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting a40d9e752934472da03f69d753a7d7d8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732666528363 2024-11-27T00:15:36,323 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5603293d8abe4d1aa6cbaba48d8c4ac9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732666529317 2024-11-27T00:15:36,324 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f32602525d4561b53bc96da2d98461, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732666530480 2024-11-27T00:15:36,324 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1941057103a446a4b424097b09931241, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732666531705 2024-11-27T00:15:36,324 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e869aa8be743f3afcf04a47a81ca31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732666533892 2024-11-27T00:15:36,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T00:15:36,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:36,328 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:15:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:36,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/9eb5af8a1a1240ab9c2a6554921894b6 is 50, key is test_row_0/A:col10/1732666535084/Put/seqid=0 2024-11-27T00:15:36,337 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#C#compaction#255 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:36,338 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1781b2dcf3c44a75a860d13756b0a901 is 50, key is test_row_0/C:col10/1732666535036/Put/seqid=0 2024-11-27T00:15:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T00:15:36,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742120_1296 (size=12301) 2024-11-27T00:15:36,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742121_1297 (size=12847) 2024-11-27T00:15:36,363 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/1781b2dcf3c44a75a860d13756b0a901 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1781b2dcf3c44a75a860d13756b0a901 2024-11-27T00:15:36,367 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/C of 40e749124bfcc05793d8d1161b1bd9fa into 1781b2dcf3c44a75a860d13756b0a901(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:36,367 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:36,367 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/C, priority=10, startTime=1732666536285; duration=0sec 2024-11-27T00:15:36,368 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:36,368 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:C 2024-11-27T00:15:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:36,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:36,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666596594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:36,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666596597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666596704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666596704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:36,724 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/b0b619b49fe84077b9c5da7b09854db1 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b0b619b49fe84077b9c5da7b09854db1 2024-11-27T00:15:36,728 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/A of 40e749124bfcc05793d8d1161b1bd9fa into b0b619b49fe84077b9c5da7b09854db1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:36,728 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:36,728 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/A, priority=10, startTime=1732666536285; duration=0sec 2024-11-27T00:15:36,728 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:36,728 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:A 2024-11-27T00:15:36,751 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/9eb5af8a1a1240ab9c2a6554921894b6 2024-11-27T00:15:36,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/4d1f2704aea6473aafe15f5669a0ea2d is 50, key is test_row_0/B:col10/1732666535084/Put/seqid=0 2024-11-27T00:15:36,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742122_1298 (size=12301) 2024-11-27T00:15:36,761 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/4d1f2704aea6473aafe15f5669a0ea2d 2024-11-27T00:15:36,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/791f83b0f7e042fdb9e4bca20255bacc is 50, key is test_row_0/C:col10/1732666535084/Put/seqid=0 2024-11-27T00:15:36,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742123_1299 (size=12301) 2024-11-27T00:15:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T00:15:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666596914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:36,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666596914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,172 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/791f83b0f7e042fdb9e4bca20255bacc 2024-11-27T00:15:37,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/9eb5af8a1a1240ab9c2a6554921894b6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/9eb5af8a1a1240ab9c2a6554921894b6 2024-11-27T00:15:37,182 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/9eb5af8a1a1240ab9c2a6554921894b6, entries=150, sequenceid=297, filesize=12.0 K 2024-11-27T00:15:37,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/4d1f2704aea6473aafe15f5669a0ea2d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4d1f2704aea6473aafe15f5669a0ea2d 2024-11-27T00:15:37,188 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4d1f2704aea6473aafe15f5669a0ea2d, entries=150, sequenceid=297, filesize=12.0 K 2024-11-27T00:15:37,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/791f83b0f7e042fdb9e4bca20255bacc as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/791f83b0f7e042fdb9e4bca20255bacc 2024-11-27T00:15:37,193 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/791f83b0f7e042fdb9e4bca20255bacc, entries=150, sequenceid=297, filesize=12.0 K 2024-11-27T00:15:37,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 40e749124bfcc05793d8d1161b1bd9fa in 866ms, sequenceid=297, compaction requested=false 2024-11-27T00:15:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-27T00:15:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-27T00:15:37,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-27T00:15:37,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4780 sec 2024-11-27T00:15:37,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4830 sec 2024-11-27T00:15:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:37,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T00:15:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:37,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/fc640a6080fe442d9b5b52d25278c336 is 50, key is test_row_0/A:col10/1732666537226/Put/seqid=0 2024-11-27T00:15:37,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742124_1300 (size=14741) 2024-11-27T00:15:37,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666597254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666597254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666597256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,260 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666597258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666597364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666597364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666597374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666597574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666597574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666597584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/fc640a6080fe442d9b5b52d25278c336 2024-11-27T00:15:37,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/b826666ec24a4c9c81a95cbbba8fe6b9 is 50, key is test_row_0/B:col10/1732666537226/Put/seqid=0 2024-11-27T00:15:37,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742125_1301 (size=12301) 2024-11-27T00:15:37,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/b826666ec24a4c9c81a95cbbba8fe6b9 2024-11-27T00:15:37,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/d92f3fe7815846efbdff3e06f4130d21 is 50, key is test_row_0/C:col10/1732666537226/Put/seqid=0 2024-11-27T00:15:37,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742126_1302 (size=12301) 2024-11-27T00:15:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T00:15:37,874 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:37,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-27T00:15:37,876 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T00:15:37,877 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:37,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:37,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666597883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666597883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666597894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:37,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T00:15:38,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T00:15:38,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:38,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:38,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:38,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:38,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:38,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/d92f3fe7815846efbdff3e06f4130d21 2024-11-27T00:15:38,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/fc640a6080fe442d9b5b52d25278c336 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fc640a6080fe442d9b5b52d25278c336 2024-11-27T00:15:38,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fc640a6080fe442d9b5b52d25278c336, entries=200, sequenceid=313, filesize=14.4 K 2024-11-27T00:15:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/b826666ec24a4c9c81a95cbbba8fe6b9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/b826666ec24a4c9c81a95cbbba8fe6b9 2024-11-27T00:15:38,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/b826666ec24a4c9c81a95cbbba8fe6b9, entries=150, sequenceid=313, filesize=12.0 K 2024-11-27T00:15:38,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/d92f3fe7815846efbdff3e06f4130d21 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/d92f3fe7815846efbdff3e06f4130d21 2024-11-27T00:15:38,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/d92f3fe7815846efbdff3e06f4130d21, entries=150, sequenceid=313, filesize=12.0 K 2024-11-27T00:15:38,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 40e749124bfcc05793d8d1161b1bd9fa in 850ms, sequenceid=313, compaction requested=true 2024-11-27T00:15:38,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:38,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:38,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:38,077 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:38,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:38,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:38,077 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:38,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:38,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:38,077 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:38,078 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/B is initiating minor compaction (all files) 2024-11-27T00:15:38,078 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/B in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:38,078 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:38,078 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/41e69e6289b94de89519c2aa43487da5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4d1f2704aea6473aafe15f5669a0ea2d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/b826666ec24a4c9c81a95cbbba8fe6b9] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.6 K 2024-11-27T00:15:38,078 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/A is initiating minor compaction (all files) 2024-11-27T00:15:38,078 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/A in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:38,078 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b0b619b49fe84077b9c5da7b09854db1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/9eb5af8a1a1240ab9c2a6554921894b6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fc640a6080fe442d9b5b52d25278c336] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=39.0 K 2024-11-27T00:15:38,079 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 41e69e6289b94de89519c2aa43487da5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732666533892 2024-11-27T00:15:38,079 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0b619b49fe84077b9c5da7b09854db1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732666533892 2024-11-27T00:15:38,079 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eb5af8a1a1240ab9c2a6554921894b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732666535084 2024-11-27T00:15:38,079 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1f2704aea6473aafe15f5669a0ea2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732666535084 2024-11-27T00:15:38,079 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc640a6080fe442d9b5b52d25278c336, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732666536589 2024-11-27T00:15:38,079 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b826666ec24a4c9c81a95cbbba8fe6b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732666536589 2024-11-27T00:15:38,085 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:38,086 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:38,086 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/d5bfe16b28c349a489bdd13573184650 is 50, key is test_row_0/A:col10/1732666537226/Put/seqid=0 2024-11-27T00:15:38,086 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/fde95b1f73b44d95aa4b22d0cbaa9a5f is 50, key is test_row_0/B:col10/1732666537226/Put/seqid=0 2024-11-27T00:15:38,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742127_1303 (size=12949) 2024-11-27T00:15:38,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742128_1304 (size=12949) 2024-11-27T00:15:38,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T00:15:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:38,182 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:15:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:38,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:38,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4dc6e0987cc24417ae929bb625698fdd is 50, key is test_row_0/A:col10/1732666537245/Put/seqid=0 2024-11-27T00:15:38,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742129_1305 (size=12301) 2024-11-27T00:15:38,191 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4dc6e0987cc24417ae929bb625698fdd 2024-11-27T00:15:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T00:15:38,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/eac63d994bc74408a1fb609f0ae6a33a is 50, key is test_row_0/B:col10/1732666537245/Put/seqid=0 2024-11-27T00:15:38,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742130_1306 (size=12301) 2024-11-27T00:15:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:38,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666598413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666598414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666598419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,494 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/fde95b1f73b44d95aa4b22d0cbaa9a5f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fde95b1f73b44d95aa4b22d0cbaa9a5f 2024-11-27T00:15:38,494 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/d5bfe16b28c349a489bdd13573184650 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/d5bfe16b28c349a489bdd13573184650 2024-11-27T00:15:38,499 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/A of 40e749124bfcc05793d8d1161b1bd9fa into d5bfe16b28c349a489bdd13573184650(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:38,499 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/B of 40e749124bfcc05793d8d1161b1bd9fa into fde95b1f73b44d95aa4b22d0cbaa9a5f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:38,499 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:38,499 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:38,499 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/A, priority=13, startTime=1732666538076; duration=0sec 2024-11-27T00:15:38,499 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/B, priority=13, startTime=1732666538077; duration=0sec 2024-11-27T00:15:38,500 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:38,500 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:38,500 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:B 2024-11-27T00:15:38,500 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:A 2024-11-27T00:15:38,500 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:38,500 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:38,500 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/C is initiating minor compaction (all files) 2024-11-27T00:15:38,501 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/C in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:38,501 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1781b2dcf3c44a75a860d13756b0a901, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/791f83b0f7e042fdb9e4bca20255bacc, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/d92f3fe7815846efbdff3e06f4130d21] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.6 K 2024-11-27T00:15:38,501 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1781b2dcf3c44a75a860d13756b0a901, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732666533892 2024-11-27T00:15:38,501 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 791f83b0f7e042fdb9e4bca20255bacc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732666535084 2024-11-27T00:15:38,501 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d92f3fe7815846efbdff3e06f4130d21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732666536589 2024-11-27T00:15:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T00:15:38,507 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#C#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:38,508 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/7b0e43f536304504ad56e15e360f402d is 50, key is test_row_0/C:col10/1732666537226/Put/seqid=0 2024-11-27T00:15:38,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742131_1307 (size=12949) 2024-11-27T00:15:38,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666598525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666598525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666598534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,608 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/eac63d994bc74408a1fb609f0ae6a33a 2024-11-27T00:15:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/3bc24d9845d54bc8bde8d697bd4ea9a5 is 50, key is test_row_0/C:col10/1732666537245/Put/seqid=0 2024-11-27T00:15:38,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742132_1308 (size=12301) 2024-11-27T00:15:38,617 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/3bc24d9845d54bc8bde8d697bd4ea9a5 2024-11-27T00:15:38,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4dc6e0987cc24417ae929bb625698fdd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4dc6e0987cc24417ae929bb625698fdd 2024-11-27T00:15:38,625 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4dc6e0987cc24417ae929bb625698fdd, entries=150, sequenceid=331, filesize=12.0 K 2024-11-27T00:15:38,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/eac63d994bc74408a1fb609f0ae6a33a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/eac63d994bc74408a1fb609f0ae6a33a 2024-11-27T00:15:38,630 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/eac63d994bc74408a1fb609f0ae6a33a, entries=150, sequenceid=331, filesize=12.0 K 2024-11-27T00:15:38,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/3bc24d9845d54bc8bde8d697bd4ea9a5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3bc24d9845d54bc8bde8d697bd4ea9a5 2024-11-27T00:15:38,633 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3bc24d9845d54bc8bde8d697bd4ea9a5, entries=150, sequenceid=331, filesize=12.0 K 2024-11-27T00:15:38,634 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 40e749124bfcc05793d8d1161b1bd9fa in 452ms, sequenceid=331, compaction requested=false 2024-11-27T00:15:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-27T00:15:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-27T00:15:38,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-27T00:15:38,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 759 msec 2024-11-27T00:15:38,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 763 msec 2024-11-27T00:15:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:38,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:15:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:38,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:38,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4e47e658f5004a80acd63fccbc9b59eb is 50, key is test_row_0/A:col10/1732666538744/Put/seqid=0 2024-11-27T00:15:38,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742133_1309 (size=14741) 2024-11-27T00:15:38,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666598768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666598769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666598770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666598874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666598884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666598884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:38,923 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/7b0e43f536304504ad56e15e360f402d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7b0e43f536304504ad56e15e360f402d 2024-11-27T00:15:38,928 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/C of 40e749124bfcc05793d8d1161b1bd9fa into 7b0e43f536304504ad56e15e360f402d(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:38,928 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:38,928 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/C, priority=13, startTime=1732666538077; duration=0sec 2024-11-27T00:15:38,928 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:38,928 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:C 2024-11-27T00:15:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T00:15:39,014 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:39,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-27T00:15:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T00:15:39,016 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:39,017 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:39,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:39,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666599083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666599094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666599094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T00:15:39,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4e47e658f5004a80acd63fccbc9b59eb 2024-11-27T00:15:39,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/f44fa7becbaf4df7a8a5bfe3b1f1430c is 50, key is test_row_0/B:col10/1732666538744/Put/seqid=0 2024-11-27T00:15:39,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742134_1310 (size=12301) 2024-11-27T00:15:39,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T00:15:39,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:39,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53390 deadline: 1732666599274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,277 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T00:15:39,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:39,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T00:15:39,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666599393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666599403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666599404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T00:15:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/f44fa7becbaf4df7a8a5bfe3b1f1430c 2024-11-27T00:15:39,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/541a786539ad44b69889b16ed931862c is 50, key is test_row_0/C:col10/1732666538744/Put/seqid=0 2024-11-27T00:15:39,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742135_1311 (size=12301) 2024-11-27T00:15:39,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T00:15:39,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:39,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T00:15:39,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T00:15:39,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:39,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666599903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666599914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666599914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:39,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T00:15:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:39,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:39,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/541a786539ad44b69889b16ed931862c 2024-11-27T00:15:39,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4e47e658f5004a80acd63fccbc9b59eb as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4e47e658f5004a80acd63fccbc9b59eb 2024-11-27T00:15:39,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4e47e658f5004a80acd63fccbc9b59eb, entries=200, sequenceid=348, filesize=14.4 K 2024-11-27T00:15:39,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/f44fa7becbaf4df7a8a5bfe3b1f1430c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/f44fa7becbaf4df7a8a5bfe3b1f1430c 2024-11-27T00:15:39,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/f44fa7becbaf4df7a8a5bfe3b1f1430c, entries=150, sequenceid=348, filesize=12.0 K 2024-11-27T00:15:39,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/541a786539ad44b69889b16ed931862c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/541a786539ad44b69889b16ed931862c 2024-11-27T00:15:39,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/541a786539ad44b69889b16ed931862c, entries=150, sequenceid=348, filesize=12.0 K 2024-11-27T00:15:39,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 40e749124bfcc05793d8d1161b1bd9fa in 1247ms, sequenceid=348, compaction requested=true 2024-11-27T00:15:39,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:39,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:39,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:39,994 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:39,994 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:39,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:39,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:39,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:39,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:39,995 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:39,995 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:39,995 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/A is initiating minor compaction (all files) 2024-11-27T00:15:39,995 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/B is initiating minor compaction (all files) 2024-11-27T00:15:39,995 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/B in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,995 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/A in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:39,995 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fde95b1f73b44d95aa4b22d0cbaa9a5f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/eac63d994bc74408a1fb609f0ae6a33a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/f44fa7becbaf4df7a8a5bfe3b1f1430c] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.7 K 2024-11-27T00:15:39,995 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/d5bfe16b28c349a489bdd13573184650, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4dc6e0987cc24417ae929bb625698fdd, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4e47e658f5004a80acd63fccbc9b59eb] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=39.1 K 2024-11-27T00:15:39,995 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5bfe16b28c349a489bdd13573184650, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732666536589 2024-11-27T00:15:39,995 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting fde95b1f73b44d95aa4b22d0cbaa9a5f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732666536589 2024-11-27T00:15:39,995 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting eac63d994bc74408a1fb609f0ae6a33a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732666537245 2024-11-27T00:15:39,995 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc6e0987cc24417ae929bb625698fdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732666537245 2024-11-27T00:15:39,996 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f44fa7becbaf4df7a8a5bfe3b1f1430c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732666538407 2024-11-27T00:15:39,996 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e47e658f5004a80acd63fccbc9b59eb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732666538406 2024-11-27T00:15:40,004 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:40,005 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/ccf04c97737c473db378e4ba5c1b3099 is 50, key is test_row_0/B:col10/1732666538744/Put/seqid=0 2024-11-27T00:15:40,006 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:40,006 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/1c748b46150f453dbf31f261ff373d82 is 50, key is test_row_0/A:col10/1732666538744/Put/seqid=0 2024-11-27T00:15:40,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742136_1312 (size=13051) 2024-11-27T00:15:40,016 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/1c748b46150f453dbf31f261ff373d82 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/1c748b46150f453dbf31f261ff373d82 2024-11-27T00:15:40,021 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/A of 40e749124bfcc05793d8d1161b1bd9fa into 1c748b46150f453dbf31f261ff373d82(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:40,022 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:40,022 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/A, priority=13, startTime=1732666539994; duration=0sec 2024-11-27T00:15:40,022 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:40,022 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:A 2024-11-27T00:15:40,022 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:40,023 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:40,023 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/C is initiating minor compaction (all files) 2024-11-27T00:15:40,023 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/C in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:40,023 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7b0e43f536304504ad56e15e360f402d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3bc24d9845d54bc8bde8d697bd4ea9a5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/541a786539ad44b69889b16ed931862c] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.7 K 2024-11-27T00:15:40,024 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0e43f536304504ad56e15e360f402d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732666536589 2024-11-27T00:15:40,024 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bc24d9845d54bc8bde8d697bd4ea9a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732666537245 2024-11-27T00:15:40,024 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 541a786539ad44b69889b16ed931862c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732666538407 2024-11-27T00:15:40,031 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:40,031 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/f160ede81c51454ba0ce5b0eca7725d5 is 50, key is test_row_0/C:col10/1732666538744/Put/seqid=0 2024-11-27T00:15:40,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742137_1313 (size=13051) 2024-11-27T00:15:40,042 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/ccf04c97737c473db378e4ba5c1b3099 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ccf04c97737c473db378e4ba5c1b3099 2024-11-27T00:15:40,050 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/B of 40e749124bfcc05793d8d1161b1bd9fa into ccf04c97737c473db378e4ba5c1b3099(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:40,050 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:40,051 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/B, priority=13, startTime=1732666539994; duration=0sec 2024-11-27T00:15:40,051 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:40,051 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:B 2024-11-27T00:15:40,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742138_1314 (size=13051) 2024-11-27T00:15:40,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-27T00:15:40,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:40,087 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:15:40,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:40,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:40,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:40,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:40,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:40,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:40,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4f71407b7bde450482fca9152b5d98e6 is 50, key is test_row_0/A:col10/1732666538768/Put/seqid=0 2024-11-27T00:15:40,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742139_1315 (size=12301) 2024-11-27T00:15:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T00:15:40,460 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/f160ede81c51454ba0ce5b0eca7725d5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/f160ede81c51454ba0ce5b0eca7725d5 2024-11-27T00:15:40,466 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/C of 40e749124bfcc05793d8d1161b1bd9fa into f160ede81c51454ba0ce5b0eca7725d5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:40,466 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:40,466 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/C, priority=13, startTime=1732666539994; duration=0sec 2024-11-27T00:15:40,466 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:40,466 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:C 2024-11-27T00:15:40,496 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4f71407b7bde450482fca9152b5d98e6 2024-11-27T00:15:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/8dce6a40efaa443da4bb6961ec352d5a is 50, key is test_row_0/B:col10/1732666538768/Put/seqid=0 2024-11-27T00:15:40,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742140_1316 (size=12301) 2024-11-27T00:15:40,911 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/8dce6a40efaa443da4bb6961ec352d5a 2024-11-27T00:15:40,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:40,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/c19bd0495beb4dd4b56dba83a2d57123 is 50, key is test_row_0/C:col10/1732666538768/Put/seqid=0 2024-11-27T00:15:40,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742141_1317 (size=12301) 2024-11-27T00:15:40,940 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/c19bd0495beb4dd4b56dba83a2d57123 2024-11-27T00:15:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/4f71407b7bde450482fca9152b5d98e6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4f71407b7bde450482fca9152b5d98e6 2024-11-27T00:15:40,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666600943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:40,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666600944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:40,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666600944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:40,951 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4f71407b7bde450482fca9152b5d98e6, entries=150, sequenceid=368, filesize=12.0 K 2024-11-27T00:15:40,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/8dce6a40efaa443da4bb6961ec352d5a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/8dce6a40efaa443da4bb6961ec352d5a 2024-11-27T00:15:40,975 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/8dce6a40efaa443da4bb6961ec352d5a, entries=150, sequenceid=368, filesize=12.0 K 2024-11-27T00:15:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/c19bd0495beb4dd4b56dba83a2d57123 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/c19bd0495beb4dd4b56dba83a2d57123 2024-11-27T00:15:40,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/c19bd0495beb4dd4b56dba83a2d57123, entries=150, sequenceid=368, filesize=12.0 K 2024-11-27T00:15:40,996 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 40e749124bfcc05793d8d1161b1bd9fa in 909ms, sequenceid=368, compaction requested=false 2024-11-27T00:15:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-27T00:15:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-27T00:15:41,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-27T00:15:41,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9810 sec 2024-11-27T00:15:41,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.9850 sec 2024-11-27T00:15:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:41,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:15:41,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:41,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:41,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:41,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:41,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:41,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:41,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/024f0bd655734ddd9f273701e8ad537e is 50, key is test_row_0/A:col10/1732666541056/Put/seqid=0 2024-11-27T00:15:41,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742142_1318 (size=17181) 2024-11-27T00:15:41,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/024f0bd655734ddd9f273701e8ad537e 2024-11-27T00:15:41,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666601092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666601095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666601097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/fcb12231aa5c42e88cb90a909d966107 is 50, key is test_row_0/B:col10/1732666541056/Put/seqid=0 2024-11-27T00:15:41,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742143_1319 (size=12301) 2024-11-27T00:15:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T00:15:41,165 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:41,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-27T00:15:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T00:15:41,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:41,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:41,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666601204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666601204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666601214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T00:15:41,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T00:15:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:41,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:41,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666601414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666601414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666601424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T00:15:41,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:41,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T00:15:41,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/fcb12231aa5c42e88cb90a909d966107 2024-11-27T00:15:41,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/041bbb93ec58458da2179528398e711d is 50, key is test_row_0/C:col10/1732666541056/Put/seqid=0 2024-11-27T00:15:41,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742144_1320 (size=12301) 2024-11-27T00:15:41,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/041bbb93ec58458da2179528398e711d 2024-11-27T00:15:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/024f0bd655734ddd9f273701e8ad537e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/024f0bd655734ddd9f273701e8ad537e 2024-11-27T00:15:41,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/024f0bd655734ddd9f273701e8ad537e, entries=250, sequenceid=387, filesize=16.8 K 2024-11-27T00:15:41,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/fcb12231aa5c42e88cb90a909d966107 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fcb12231aa5c42e88cb90a909d966107 2024-11-27T00:15:41,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fcb12231aa5c42e88cb90a909d966107, entries=150, sequenceid=387, filesize=12.0 K 2024-11-27T00:15:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/041bbb93ec58458da2179528398e711d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/041bbb93ec58458da2179528398e711d 2024-11-27T00:15:41,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/041bbb93ec58458da2179528398e711d, entries=150, sequenceid=387, filesize=12.0 K 2024-11-27T00:15:41,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 40e749124bfcc05793d8d1161b1bd9fa in 564ms, sequenceid=387, compaction requested=true 2024-11-27T00:15:41,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:41,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:41,621 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:41,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:41,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:41,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:41,621 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:41,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:41,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/B is initiating minor compaction (all files) 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/A is initiating minor compaction (all files) 2024-11-27T00:15:41,622 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/B in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:41,622 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/A in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:41,622 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ccf04c97737c473db378e4ba5c1b3099, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/8dce6a40efaa443da4bb6961ec352d5a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fcb12231aa5c42e88cb90a909d966107] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.8 K 2024-11-27T00:15:41,622 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/1c748b46150f453dbf31f261ff373d82, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4f71407b7bde450482fca9152b5d98e6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/024f0bd655734ddd9f273701e8ad537e] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=41.5 K 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ccf04c97737c473db378e4ba5c1b3099, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732666538407 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c748b46150f453dbf31f261ff373d82, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732666538407 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dce6a40efaa443da4bb6961ec352d5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732666538762 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f71407b7bde450482fca9152b5d98e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732666538762 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb12231aa5c42e88cb90a909d966107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732666540933 2024-11-27T00:15:41,622 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024f0bd655734ddd9f273701e8ad537e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732666540926 2024-11-27T00:15:41,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-27T00:15:41,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:41,630 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:15:41,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:41,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:41,636 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:41,636 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/6f0ef3d5a4e4463ab14c804f77441542 is 50, key is test_row_0/A:col10/1732666541056/Put/seqid=0 2024-11-27T00:15:41,638 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:41,639 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/879e82c0eb4048369eba680dd03eaa52 is 50, key is test_row_0/B:col10/1732666541056/Put/seqid=0 2024-11-27T00:15:41,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/acfbb417e13440dd81a9604b699c9d31 is 50, key is test_row_0/A:col10/1732666541090/Put/seqid=0 2024-11-27T00:15:41,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742146_1322 (size=13153) 2024-11-27T00:15:41,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742145_1321 (size=13153) 2024-11-27T00:15:41,659 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/6f0ef3d5a4e4463ab14c804f77441542 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6f0ef3d5a4e4463ab14c804f77441542 2024-11-27T00:15:41,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742147_1323 (size=12301) 2024-11-27T00:15:41,666 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/acfbb417e13440dd81a9604b699c9d31 2024-11-27T00:15:41,668 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/A of 40e749124bfcc05793d8d1161b1bd9fa into 6f0ef3d5a4e4463ab14c804f77441542(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:41,668 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:41,668 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/A, priority=13, startTime=1732666541621; duration=0sec 2024-11-27T00:15:41,668 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:41,668 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:A 2024-11-27T00:15:41,668 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:41,670 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:41,670 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/C is initiating minor compaction (all files) 2024-11-27T00:15:41,670 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/C in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:41,670 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/f160ede81c51454ba0ce5b0eca7725d5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/c19bd0495beb4dd4b56dba83a2d57123, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/041bbb93ec58458da2179528398e711d] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.8 K 2024-11-27T00:15:41,671 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f160ede81c51454ba0ce5b0eca7725d5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732666538407 2024-11-27T00:15:41,672 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19bd0495beb4dd4b56dba83a2d57123, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732666538762 2024-11-27T00:15:41,672 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 041bbb93ec58458da2179528398e711d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732666540933 2024-11-27T00:15:41,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/48e34cd6abae4799a556eb09c875205c is 50, key is test_row_0/B:col10/1732666541090/Put/seqid=0 2024-11-27T00:15:41,683 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#C#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:41,683 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/61c6c4aa99764c558ec94a980620d84a is 50, key is test_row_0/C:col10/1732666541056/Put/seqid=0 2024-11-27T00:15:41,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742148_1324 (size=12301) 2024-11-27T00:15:41,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742149_1325 (size=13153) 2024-11-27T00:15:41,706 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/61c6c4aa99764c558ec94a980620d84a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/61c6c4aa99764c558ec94a980620d84a 2024-11-27T00:15:41,711 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/C of 40e749124bfcc05793d8d1161b1bd9fa into 61c6c4aa99764c558ec94a980620d84a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:41,711 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:41,711 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/C, priority=13, startTime=1732666541621; duration=0sec 2024-11-27T00:15:41,711 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:41,711 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:C 2024-11-27T00:15:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:41,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. as already flushing 2024-11-27T00:15:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666601747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666601748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666601749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T00:15:41,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666601854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666601863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666601863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:42,057 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/879e82c0eb4048369eba680dd03eaa52 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/879e82c0eb4048369eba680dd03eaa52 2024-11-27T00:15:42,062 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/B of 40e749124bfcc05793d8d1161b1bd9fa into 879e82c0eb4048369eba680dd03eaa52(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:42,062 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:42,062 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/B, priority=13, startTime=1732666541621; duration=0sec 2024-11-27T00:15:42,062 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:42,062 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:B 2024-11-27T00:15:42,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:42,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666602074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666602074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:42,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666602074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:42,099 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/48e34cd6abae4799a556eb09c875205c 2024-11-27T00:15:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/420c9096c53c41efbb6853c4fe93198e is 50, key is test_row_0/C:col10/1732666541090/Put/seqid=0 2024-11-27T00:15:42,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742150_1326 (size=12301) 2024-11-27T00:15:42,121 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T00:15:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T00:15:42,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53354 deadline: 1732666602313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:42,316 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 40e749124bfcc05793d8d1161b1bd9fa of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:15:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:42,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53296 deadline: 1732666602384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53344 deadline: 1732666602384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53298 deadline: 1732666602384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:15:42,510 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/420c9096c53c41efbb6853c4fe93198e 2024-11-27T00:15:42,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/acfbb417e13440dd81a9604b699c9d31 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/acfbb417e13440dd81a9604b699c9d31 2024-11-27T00:15:42,521 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/acfbb417e13440dd81a9604b699c9d31, entries=150, sequenceid=402, filesize=12.0 K 2024-11-27T00:15:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/48e34cd6abae4799a556eb09c875205c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/48e34cd6abae4799a556eb09c875205c 2024-11-27T00:15:42,530 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/48e34cd6abae4799a556eb09c875205c, entries=150, sequenceid=402, filesize=12.0 K 2024-11-27T00:15:42,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/420c9096c53c41efbb6853c4fe93198e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/420c9096c53c41efbb6853c4fe93198e 2024-11-27T00:15:42,535 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/420c9096c53c41efbb6853c4fe93198e, entries=150, sequenceid=402, filesize=12.0 K 2024-11-27T00:15:42,536 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 40e749124bfcc05793d8d1161b1bd9fa in 906ms, sequenceid=402, compaction requested=false 2024-11-27T00:15:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-27T00:15:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-27T00:15:42,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-27T00:15:42,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3690 sec 2024-11-27T00:15:42,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.3750 sec 2024-11-27T00:15:42,812 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-27T00:15:42,812 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:42,812 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,812 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-27T00:15:42,812 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:42,812 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:42,812 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:42,812 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,813 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-27T00:15:42,813 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:42,813 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,813 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,813 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:42,815 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-27T00:15:42,815 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:42,815 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,815 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:42,816 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-11-27T00:15:42,816 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:42,816 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,816 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:42,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:15:42,896 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-27T00:15:42,896 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:42,896 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,896 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-27T00:15:42,896 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,896 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:42,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:42,896 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,896 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:42,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:42,897 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-27T00:15:42,897 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:42,897 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,898 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:42,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:42,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/dd674731925246428b6d885651b61914 is 50, key is test_row_0/A:col10/1732666542894/Put/seqid=0 2024-11-27T00:15:42,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742151_1327 (size=12301) 2024-11-27T00:15:42,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/dd674731925246428b6d885651b61914 2024-11-27T00:15:42,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/e05d849bf6ca4ca7a23f26fb8ec8d630 is 50, key is test_row_0/B:col10/1732666542894/Put/seqid=0 2024-11-27T00:15:42,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742152_1328 (size=12301) 2024-11-27T00:15:43,306 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-27T00:15:43,306 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:43,306 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:43,306 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:43,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T00:15:43,315 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:43,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/e05d849bf6ca4ca7a23f26fb8ec8d630 2024-11-27T00:15:43,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/0777044e79f8456b89ca51812cf23aef is 50, key is test_row_0/C:col10/1732666542894/Put/seqid=0 2024-11-27T00:15:43,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742153_1329 (size=12301) 2024-11-27T00:15:43,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/0777044e79f8456b89ca51812cf23aef 2024-11-27T00:15:43,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/dd674731925246428b6d885651b61914 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/dd674731925246428b6d885651b61914 2024-11-27T00:15:43,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/dd674731925246428b6d885651b61914, entries=150, sequenceid=425, filesize=12.0 K 2024-11-27T00:15:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/e05d849bf6ca4ca7a23f26fb8ec8d630 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/e05d849bf6ca4ca7a23f26fb8ec8d630 2024-11-27T00:15:43,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/e05d849bf6ca4ca7a23f26fb8ec8d630, entries=150, sequenceid=425, filesize=12.0 K 2024-11-27T00:15:43,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/0777044e79f8456b89ca51812cf23aef as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/0777044e79f8456b89ca51812cf23aef 2024-11-27T00:15:43,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/0777044e79f8456b89ca51812cf23aef, entries=150, sequenceid=425, filesize=12.0 K 2024-11-27T00:15:43,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 40e749124bfcc05793d8d1161b1bd9fa in 865ms, sequenceid=425, compaction requested=true 2024-11-27T00:15:43,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:43,760 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:15:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:43,761 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:15:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 40e749124bfcc05793d8d1161b1bd9fa:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:15:43,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:43,762 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:43,762 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:43,762 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/B is initiating minor compaction (all files) 2024-11-27T00:15:43,762 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/A is initiating minor compaction (all files) 2024-11-27T00:15:43,762 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/B in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:43,762 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/A in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:43,762 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/879e82c0eb4048369eba680dd03eaa52, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/48e34cd6abae4799a556eb09c875205c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/e05d849bf6ca4ca7a23f26fb8ec8d630] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.9 K 2024-11-27T00:15:43,762 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6f0ef3d5a4e4463ab14c804f77441542, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/acfbb417e13440dd81a9604b699c9d31, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/dd674731925246428b6d885651b61914] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.9 K 2024-11-27T00:15:43,762 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f0ef3d5a4e4463ab14c804f77441542, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732666540933 2024-11-27T00:15:43,762 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 879e82c0eb4048369eba680dd03eaa52, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732666540933 2024-11-27T00:15:43,763 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting acfbb417e13440dd81a9604b699c9d31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732666541090 2024-11-27T00:15:43,763 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 48e34cd6abae4799a556eb09c875205c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732666541090 2024-11-27T00:15:43,763 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd674731925246428b6d885651b61914, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732666541746 2024-11-27T00:15:43,763 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e05d849bf6ca4ca7a23f26fb8ec8d630, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732666541746 2024-11-27T00:15:43,773 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#B#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:43,773 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#A#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:43,774 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/1f4f4441043443c5b667ed7ac46d37de is 50, key is test_row_0/A:col10/1732666542894/Put/seqid=0 2024-11-27T00:15:43,774 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/2aadd32855f34bb69e014b76fcffa03b is 50, key is test_row_0/B:col10/1732666542894/Put/seqid=0 2024-11-27T00:15:43,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742154_1330 (size=13255) 2024-11-27T00:15:43,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742155_1331 (size=13255) 2024-11-27T00:15:44,191 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/2aadd32855f34bb69e014b76fcffa03b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/2aadd32855f34bb69e014b76fcffa03b 2024-11-27T00:15:44,191 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/1f4f4441043443c5b667ed7ac46d37de as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/1f4f4441043443c5b667ed7ac46d37de 2024-11-27T00:15:44,195 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/B of 40e749124bfcc05793d8d1161b1bd9fa into 2aadd32855f34bb69e014b76fcffa03b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:44,195 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/A of 40e749124bfcc05793d8d1161b1bd9fa into 1f4f4441043443c5b667ed7ac46d37de(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:44,195 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:44,195 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:44,195 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/B, priority=13, startTime=1732666543761; duration=0sec 2024-11-27T00:15:44,195 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/A, priority=13, startTime=1732666543760; duration=0sec 2024-11-27T00:15:44,195 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:44,195 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:15:44,195 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:B 2024-11-27T00:15:44,195 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:A 2024-11-27T00:15:44,195 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:15:44,196 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:15:44,196 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 40e749124bfcc05793d8d1161b1bd9fa/C is initiating minor compaction (all files) 2024-11-27T00:15:44,196 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 40e749124bfcc05793d8d1161b1bd9fa/C in TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:44,197 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/61c6c4aa99764c558ec94a980620d84a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/420c9096c53c41efbb6853c4fe93198e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/0777044e79f8456b89ca51812cf23aef] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp, totalSize=36.9 K 2024-11-27T00:15:44,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 61c6c4aa99764c558ec94a980620d84a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732666540933 2024-11-27T00:15:44,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 420c9096c53c41efbb6853c4fe93198e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732666541090 2024-11-27T00:15:44,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0777044e79f8456b89ca51812cf23aef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732666541746 2024-11-27T00:15:44,204 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 40e749124bfcc05793d8d1161b1bd9fa#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:15:44,204 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/afda804575a242d0938c54b05cbf3d38 is 50, key is test_row_0/C:col10/1732666542894/Put/seqid=0 2024-11-27T00:15:44,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742156_1332 (size=13255) 2024-11-27T00:15:44,612 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/afda804575a242d0938c54b05cbf3d38 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/afda804575a242d0938c54b05cbf3d38 2024-11-27T00:15:44,616 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40e749124bfcc05793d8d1161b1bd9fa/C of 40e749124bfcc05793d8d1161b1bd9fa into afda804575a242d0938c54b05cbf3d38(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:15:44,616 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 40e749124bfcc05793d8d1161b1bd9fa: 2024-11-27T00:15:44,616 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., storeName=40e749124bfcc05793d8d1161b1bd9fa/C, priority=13, startTime=1732666543761; duration=0sec 2024-11-27T00:15:44,617 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:15:44,617 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 40e749124bfcc05793d8d1161b1bd9fa:C 2024-11-27T00:15:52,416 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-27T00:15:52,417 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:15:52,417 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:52,417 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:52,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3038 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3045 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3026 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3063 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3027 2024-11-27T00:15:52,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T00:15:52,418 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T00:15:52,418 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:15:52,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:52,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:52,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:15:52,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T00:15:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T00:15:52,425 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666552424"}]},"ts":"1732666552424"} 2024-11-27T00:15:52,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T00:15:52,427 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T00:15:52,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:15:52,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, UNASSIGN}] 2024-11-27T00:15:52,431 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, UNASSIGN 2024-11-27T00:15:52,432 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=40e749124bfcc05793d8d1161b1bd9fa, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:52,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, UNASSIGN because future has completed 2024-11-27T00:15:52,433 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:15:52,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:15:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T00:15:52,586 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:52,586 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:15:52,586 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 40e749124bfcc05793d8d1161b1bd9fa, disabling compactions & flushes 2024-11-27T00:15:52,586 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:52,586 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:52,587 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. after waiting 0 ms 2024-11-27T00:15:52,587 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:52,587 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 40e749124bfcc05793d8d1161b1bd9fa 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-27T00:15:52,587 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=A 2024-11-27T00:15:52,587 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:52,587 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=B 2024-11-27T00:15:52,587 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:52,587 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 40e749124bfcc05793d8d1161b1bd9fa, store=C 2024-11-27T00:15:52,588 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:52,597 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/5ef2bc6554604256a466fee0534e7561 is 50, key is test_row_2/A:col10/1732666552414/Put/seqid=0 2024-11-27T00:15:52,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742157_1333 (size=7415) 2024-11-27T00:15:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T00:15:53,002 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/5ef2bc6554604256a466fee0534e7561 2024-11-27T00:15:53,010 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/538c3528f49649fcb8119bb771141789 is 50, key is test_row_2/B:col10/1732666552414/Put/seqid=0 2024-11-27T00:15:53,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742158_1334 (size=7415) 2024-11-27T00:15:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T00:15:53,417 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/538c3528f49649fcb8119bb771141789 2024-11-27T00:15:53,430 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/7433fbabe2704380bc143c552485ddef is 50, key is test_row_2/C:col10/1732666552414/Put/seqid=0 2024-11-27T00:15:53,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742159_1335 (size=7415) 2024-11-27T00:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T00:15:53,834 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/7433fbabe2704380bc143c552485ddef 2024-11-27T00:15:53,838 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/A/5ef2bc6554604256a466fee0534e7561 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/5ef2bc6554604256a466fee0534e7561 2024-11-27T00:15:53,841 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/5ef2bc6554604256a466fee0534e7561, entries=50, sequenceid=433, filesize=7.2 K 2024-11-27T00:15:53,842 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/B/538c3528f49649fcb8119bb771141789 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/538c3528f49649fcb8119bb771141789 2024-11-27T00:15:53,845 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/538c3528f49649fcb8119bb771141789, entries=50, sequenceid=433, filesize=7.2 K 2024-11-27T00:15:53,846 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/.tmp/C/7433fbabe2704380bc143c552485ddef as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7433fbabe2704380bc143c552485ddef 2024-11-27T00:15:53,849 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7433fbabe2704380bc143c552485ddef, entries=50, sequenceid=433, filesize=7.2 K 2024-11-27T00:15:53,850 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 40e749124bfcc05793d8d1161b1bd9fa in 1263ms, sequenceid=433, compaction requested=false 2024-11-27T00:15:53,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/ee04ba7bf43340c693f1269652dcd671, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6eee32d620bf40c18538f420516d4bbf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/f770e9e33ffd4d4db2ea088208c01105, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fac0f2729e8a4883bd7249c7a5908d10, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/60bbdb06891a4f1998bbf7dde57d689e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/643ae48a15c142159c1e1526da52d547, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/564b86bafed5402790f51ed42df223a5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3d955d029b69466b98d7113a5f84afe5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4db05d6b41574eecba060afd0a9d9052, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/248953ac0287442abc40dce3a16eb240, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/7819879e20eb47cca185246f2329a0f5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b69360f6126e443788b98d321e705f69, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/bc6b386dcec442f19fb365a0846432af, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/832ad8fa87ac435b8f09fa8ff14b94da, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3093e375875c4e9fa2e07e2dd9bca6b4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/790fa936687540ceb9783f8541597250, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b0b619b49fe84077b9c5da7b09854db1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/9eb5af8a1a1240ab9c2a6554921894b6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fc640a6080fe442d9b5b52d25278c336, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/d5bfe16b28c349a489bdd13573184650, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4dc6e0987cc24417ae929bb625698fdd, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4e47e658f5004a80acd63fccbc9b59eb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/1c748b46150f453dbf31f261ff373d82, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4f71407b7bde450482fca9152b5d98e6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/024f0bd655734ddd9f273701e8ad537e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6f0ef3d5a4e4463ab14c804f77441542, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/acfbb417e13440dd81a9604b699c9d31, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/dd674731925246428b6d885651b61914] to archive 2024-11-27T00:15:53,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:15:53,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/ee04ba7bf43340c693f1269652dcd671 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/ee04ba7bf43340c693f1269652dcd671 2024-11-27T00:15:53,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6eee32d620bf40c18538f420516d4bbf to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6eee32d620bf40c18538f420516d4bbf 2024-11-27T00:15:53,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/f770e9e33ffd4d4db2ea088208c01105 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/f770e9e33ffd4d4db2ea088208c01105 2024-11-27T00:15:53,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fac0f2729e8a4883bd7249c7a5908d10 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fac0f2729e8a4883bd7249c7a5908d10 2024-11-27T00:15:53,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/60bbdb06891a4f1998bbf7dde57d689e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/60bbdb06891a4f1998bbf7dde57d689e 2024-11-27T00:15:53,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/643ae48a15c142159c1e1526da52d547 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/643ae48a15c142159c1e1526da52d547 2024-11-27T00:15:53,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/564b86bafed5402790f51ed42df223a5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/564b86bafed5402790f51ed42df223a5 2024-11-27T00:15:53,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3d955d029b69466b98d7113a5f84afe5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3d955d029b69466b98d7113a5f84afe5 2024-11-27T00:15:53,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4db05d6b41574eecba060afd0a9d9052 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4db05d6b41574eecba060afd0a9d9052 2024-11-27T00:15:53,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/248953ac0287442abc40dce3a16eb240 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/248953ac0287442abc40dce3a16eb240 2024-11-27T00:15:53,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/7819879e20eb47cca185246f2329a0f5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/7819879e20eb47cca185246f2329a0f5 2024-11-27T00:15:53,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b69360f6126e443788b98d321e705f69 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b69360f6126e443788b98d321e705f69 2024-11-27T00:15:53,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/bc6b386dcec442f19fb365a0846432af to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/bc6b386dcec442f19fb365a0846432af 2024-11-27T00:15:53,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/832ad8fa87ac435b8f09fa8ff14b94da to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/832ad8fa87ac435b8f09fa8ff14b94da 2024-11-27T00:15:53,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3093e375875c4e9fa2e07e2dd9bca6b4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/3093e375875c4e9fa2e07e2dd9bca6b4 2024-11-27T00:15:53,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/790fa936687540ceb9783f8541597250 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/790fa936687540ceb9783f8541597250 2024-11-27T00:15:53,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b0b619b49fe84077b9c5da7b09854db1 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/b0b619b49fe84077b9c5da7b09854db1 2024-11-27T00:15:53,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/9eb5af8a1a1240ab9c2a6554921894b6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/9eb5af8a1a1240ab9c2a6554921894b6 2024-11-27T00:15:53,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fc640a6080fe442d9b5b52d25278c336 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/fc640a6080fe442d9b5b52d25278c336 2024-11-27T00:15:53,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/d5bfe16b28c349a489bdd13573184650 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/d5bfe16b28c349a489bdd13573184650 2024-11-27T00:15:53,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4dc6e0987cc24417ae929bb625698fdd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4dc6e0987cc24417ae929bb625698fdd 2024-11-27T00:15:53,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4e47e658f5004a80acd63fccbc9b59eb to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4e47e658f5004a80acd63fccbc9b59eb 2024-11-27T00:15:53,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/1c748b46150f453dbf31f261ff373d82 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/1c748b46150f453dbf31f261ff373d82 2024-11-27T00:15:53,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4f71407b7bde450482fca9152b5d98e6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/4f71407b7bde450482fca9152b5d98e6 2024-11-27T00:15:53,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/024f0bd655734ddd9f273701e8ad537e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/024f0bd655734ddd9f273701e8ad537e 2024-11-27T00:15:53,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6f0ef3d5a4e4463ab14c804f77441542 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/6f0ef3d5a4e4463ab14c804f77441542 2024-11-27T00:15:53,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/acfbb417e13440dd81a9604b699c9d31 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/acfbb417e13440dd81a9604b699c9d31 2024-11-27T00:15:53,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/dd674731925246428b6d885651b61914 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/dd674731925246428b6d885651b61914 2024-11-27T00:15:53,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/cb915c7d49704cbb8dd379b065661910, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/dfac0a77d38a4da89778d3ec1958a1d9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4c07b57b7da140fc96b8ecfbfd6c930a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/187aa9a7728842df8f5b5f67d16680d9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/63c9b958fcb14528bdac7236224b58f5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/78e027de517b4b57bcc72f8238df91f0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ed2ae9562abd40db83a9c4a716301704, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/864d05f5af7141109eda1734ba0e2f72, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5427558bec4d418dbff50a41022b4579, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/352a245b38c64c9c8463b9aa36dcec93, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/29236f7c15364fc2a67a57b7eb22603d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/584cb0af91ef4f39b92400b6a2b50bdf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/6d668bd9ef774918b58d64398c6f4ffb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5a1eec7168d04e80ab41e33b0c4a0f6b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5ded6f1e54bb4eb89e703966c477f2ff, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/41e69e6289b94de89519c2aa43487da5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4313223a0b6446df9ad04c9725235ce3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4d1f2704aea6473aafe15f5669a0ea2d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fde95b1f73b44d95aa4b22d0cbaa9a5f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/b826666ec24a4c9c81a95cbbba8fe6b9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/eac63d994bc74408a1fb609f0ae6a33a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ccf04c97737c473db378e4ba5c1b3099, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/f44fa7becbaf4df7a8a5bfe3b1f1430c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/8dce6a40efaa443da4bb6961ec352d5a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/879e82c0eb4048369eba680dd03eaa52, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fcb12231aa5c42e88cb90a909d966107, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/48e34cd6abae4799a556eb09c875205c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/e05d849bf6ca4ca7a23f26fb8ec8d630] to archive 2024-11-27T00:15:53,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:15:53,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/cb915c7d49704cbb8dd379b065661910 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/cb915c7d49704cbb8dd379b065661910 2024-11-27T00:15:53,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/dfac0a77d38a4da89778d3ec1958a1d9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/dfac0a77d38a4da89778d3ec1958a1d9 2024-11-27T00:15:53,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4c07b57b7da140fc96b8ecfbfd6c930a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4c07b57b7da140fc96b8ecfbfd6c930a 2024-11-27T00:15:53,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/187aa9a7728842df8f5b5f67d16680d9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/187aa9a7728842df8f5b5f67d16680d9 2024-11-27T00:15:53,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/63c9b958fcb14528bdac7236224b58f5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/63c9b958fcb14528bdac7236224b58f5 2024-11-27T00:15:53,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/78e027de517b4b57bcc72f8238df91f0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/78e027de517b4b57bcc72f8238df91f0 2024-11-27T00:15:53,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ed2ae9562abd40db83a9c4a716301704 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ed2ae9562abd40db83a9c4a716301704 2024-11-27T00:15:53,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/864d05f5af7141109eda1734ba0e2f72 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/864d05f5af7141109eda1734ba0e2f72 2024-11-27T00:15:53,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5427558bec4d418dbff50a41022b4579 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5427558bec4d418dbff50a41022b4579 2024-11-27T00:15:53,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/352a245b38c64c9c8463b9aa36dcec93 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/352a245b38c64c9c8463b9aa36dcec93 2024-11-27T00:15:53,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/29236f7c15364fc2a67a57b7eb22603d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/29236f7c15364fc2a67a57b7eb22603d 2024-11-27T00:15:53,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/584cb0af91ef4f39b92400b6a2b50bdf to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/584cb0af91ef4f39b92400b6a2b50bdf 2024-11-27T00:15:53,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/6d668bd9ef774918b58d64398c6f4ffb to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/6d668bd9ef774918b58d64398c6f4ffb 2024-11-27T00:15:53,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5a1eec7168d04e80ab41e33b0c4a0f6b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5a1eec7168d04e80ab41e33b0c4a0f6b 2024-11-27T00:15:53,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5ded6f1e54bb4eb89e703966c477f2ff to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/5ded6f1e54bb4eb89e703966c477f2ff 2024-11-27T00:15:53,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/41e69e6289b94de89519c2aa43487da5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/41e69e6289b94de89519c2aa43487da5 2024-11-27T00:15:53,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4313223a0b6446df9ad04c9725235ce3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4313223a0b6446df9ad04c9725235ce3 2024-11-27T00:15:53,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4d1f2704aea6473aafe15f5669a0ea2d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/4d1f2704aea6473aafe15f5669a0ea2d 2024-11-27T00:15:53,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fde95b1f73b44d95aa4b22d0cbaa9a5f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fde95b1f73b44d95aa4b22d0cbaa9a5f 2024-11-27T00:15:53,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/b826666ec24a4c9c81a95cbbba8fe6b9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/b826666ec24a4c9c81a95cbbba8fe6b9 2024-11-27T00:15:53,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/eac63d994bc74408a1fb609f0ae6a33a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/eac63d994bc74408a1fb609f0ae6a33a 2024-11-27T00:15:53,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ccf04c97737c473db378e4ba5c1b3099 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/ccf04c97737c473db378e4ba5c1b3099 2024-11-27T00:15:53,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/f44fa7becbaf4df7a8a5bfe3b1f1430c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/f44fa7becbaf4df7a8a5bfe3b1f1430c 2024-11-27T00:15:53,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/8dce6a40efaa443da4bb6961ec352d5a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/8dce6a40efaa443da4bb6961ec352d5a 2024-11-27T00:15:53,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/879e82c0eb4048369eba680dd03eaa52 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/879e82c0eb4048369eba680dd03eaa52 2024-11-27T00:15:53,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fcb12231aa5c42e88cb90a909d966107 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/fcb12231aa5c42e88cb90a909d966107 2024-11-27T00:15:53,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/48e34cd6abae4799a556eb09c875205c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/48e34cd6abae4799a556eb09c875205c 2024-11-27T00:15:53,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/e05d849bf6ca4ca7a23f26fb8ec8d630 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/e05d849bf6ca4ca7a23f26fb8ec8d630 2024-11-27T00:15:53,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/ee48bd9bbf604c1b84c12b45b95970c1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5ad39b944d6e46efb240229fdcb98da4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a775c9270c87419a8edae3c23165ac99, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1d66f2af1ccc4bb1b17acf06a24bd7c0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1105579bb2734c62a892a69056492a9a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/81dbb1e50a924a888e683ad02aae9d1c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/6e247d54823345c184af508902c60e7e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/140eb61cdd484d7f90b84398a0d1a993, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3fd7144dbc744e8f86bb0cacf9f7543b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b5b0e753016e46849f9c9a950f6a48fc, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5b492e4b398d45ac8f51d64d5fd3af0a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a40d9e752934472da03f69d753a7d7d8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5603293d8abe4d1aa6cbaba48d8c4ac9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/58f32602525d4561b53bc96da2d98461, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1941057103a446a4b424097b09931241, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1781b2dcf3c44a75a860d13756b0a901, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b2e869aa8be743f3afcf04a47a81ca31, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/791f83b0f7e042fdb9e4bca20255bacc, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7b0e43f536304504ad56e15e360f402d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/d92f3fe7815846efbdff3e06f4130d21, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3bc24d9845d54bc8bde8d697bd4ea9a5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/f160ede81c51454ba0ce5b0eca7725d5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/541a786539ad44b69889b16ed931862c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/c19bd0495beb4dd4b56dba83a2d57123, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/61c6c4aa99764c558ec94a980620d84a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/041bbb93ec58458da2179528398e711d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/420c9096c53c41efbb6853c4fe93198e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/0777044e79f8456b89ca51812cf23aef] to archive 2024-11-27T00:15:53,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:15:53,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/ee48bd9bbf604c1b84c12b45b95970c1 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/ee48bd9bbf604c1b84c12b45b95970c1 2024-11-27T00:15:53,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5ad39b944d6e46efb240229fdcb98da4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5ad39b944d6e46efb240229fdcb98da4 2024-11-27T00:15:53,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a775c9270c87419a8edae3c23165ac99 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a775c9270c87419a8edae3c23165ac99 2024-11-27T00:15:53,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1d66f2af1ccc4bb1b17acf06a24bd7c0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1d66f2af1ccc4bb1b17acf06a24bd7c0 2024-11-27T00:15:53,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1105579bb2734c62a892a69056492a9a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1105579bb2734c62a892a69056492a9a 2024-11-27T00:15:53,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/81dbb1e50a924a888e683ad02aae9d1c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/81dbb1e50a924a888e683ad02aae9d1c 2024-11-27T00:15:53,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/6e247d54823345c184af508902c60e7e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/6e247d54823345c184af508902c60e7e 2024-11-27T00:15:53,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/140eb61cdd484d7f90b84398a0d1a993 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/140eb61cdd484d7f90b84398a0d1a993 2024-11-27T00:15:53,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3fd7144dbc744e8f86bb0cacf9f7543b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3fd7144dbc744e8f86bb0cacf9f7543b 2024-11-27T00:15:53,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b5b0e753016e46849f9c9a950f6a48fc to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b5b0e753016e46849f9c9a950f6a48fc 2024-11-27T00:15:53,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5b492e4b398d45ac8f51d64d5fd3af0a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5b492e4b398d45ac8f51d64d5fd3af0a 2024-11-27T00:15:53,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a40d9e752934472da03f69d753a7d7d8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/a40d9e752934472da03f69d753a7d7d8 2024-11-27T00:15:53,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5603293d8abe4d1aa6cbaba48d8c4ac9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/5603293d8abe4d1aa6cbaba48d8c4ac9 2024-11-27T00:15:53,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/58f32602525d4561b53bc96da2d98461 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/58f32602525d4561b53bc96da2d98461 2024-11-27T00:15:53,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1941057103a446a4b424097b09931241 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1941057103a446a4b424097b09931241 2024-11-27T00:15:53,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1781b2dcf3c44a75a860d13756b0a901 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/1781b2dcf3c44a75a860d13756b0a901 2024-11-27T00:15:53,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b2e869aa8be743f3afcf04a47a81ca31 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/b2e869aa8be743f3afcf04a47a81ca31 2024-11-27T00:15:53,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/791f83b0f7e042fdb9e4bca20255bacc to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/791f83b0f7e042fdb9e4bca20255bacc 2024-11-27T00:15:53,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7b0e43f536304504ad56e15e360f402d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7b0e43f536304504ad56e15e360f402d 2024-11-27T00:15:53,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/d92f3fe7815846efbdff3e06f4130d21 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/d92f3fe7815846efbdff3e06f4130d21 2024-11-27T00:15:53,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3bc24d9845d54bc8bde8d697bd4ea9a5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/3bc24d9845d54bc8bde8d697bd4ea9a5 2024-11-27T00:15:53,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/f160ede81c51454ba0ce5b0eca7725d5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/f160ede81c51454ba0ce5b0eca7725d5 2024-11-27T00:15:53,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/541a786539ad44b69889b16ed931862c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/541a786539ad44b69889b16ed931862c 2024-11-27T00:15:53,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/c19bd0495beb4dd4b56dba83a2d57123 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/c19bd0495beb4dd4b56dba83a2d57123 2024-11-27T00:15:53,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/61c6c4aa99764c558ec94a980620d84a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/61c6c4aa99764c558ec94a980620d84a 2024-11-27T00:15:53,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/041bbb93ec58458da2179528398e711d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/041bbb93ec58458da2179528398e711d 2024-11-27T00:15:53,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/420c9096c53c41efbb6853c4fe93198e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/420c9096c53c41efbb6853c4fe93198e 2024-11-27T00:15:53,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/0777044e79f8456b89ca51812cf23aef to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/0777044e79f8456b89ca51812cf23aef 2024-11-27T00:15:53,931 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=1 2024-11-27T00:15:53,932 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. 2024-11-27T00:15:53,932 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 40e749124bfcc05793d8d1161b1bd9fa: Waiting for close lock at 1732666552586Running coprocessor pre-close hooks at 1732666552586Disabling compacts and flushes for region at 1732666552586Disabling writes for close at 1732666552587 (+1 ms)Obtaining lock to block concurrent updates at 1732666552587Preparing flush snapshotting stores in 40e749124bfcc05793d8d1161b1bd9fa at 1732666552587Finished memstore snapshotting TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732666552588 (+1 ms)Flushing stores of TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa. at 1732666552589 (+1 ms)Flushing 40e749124bfcc05793d8d1161b1bd9fa/A: creating writer at 1732666552589Flushing 40e749124bfcc05793d8d1161b1bd9fa/A: appending metadata at 1732666552595 (+6 ms)Flushing 40e749124bfcc05793d8d1161b1bd9fa/A: closing flushed file at 1732666552595Flushing 40e749124bfcc05793d8d1161b1bd9fa/B: creating writer at 1732666553007 (+412 ms)Flushing 40e749124bfcc05793d8d1161b1bd9fa/B: appending metadata at 1732666553009 (+2 ms)Flushing 40e749124bfcc05793d8d1161b1bd9fa/B: closing flushed file at 1732666553009Flushing 40e749124bfcc05793d8d1161b1bd9fa/C: creating writer at 1732666553424 (+415 ms)Flushing 40e749124bfcc05793d8d1161b1bd9fa/C: appending metadata at 1732666553429 (+5 ms)Flushing 40e749124bfcc05793d8d1161b1bd9fa/C: closing flushed file at 1732666553429Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2666af6b: reopening flushed file at 1732666553838 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c242c0d: reopening flushed file at 1732666553841 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d8b0073: reopening flushed file at 1732666553845 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 40e749124bfcc05793d8d1161b1bd9fa in 1263ms, sequenceid=433, compaction requested=false at 1732666553850 (+5 ms)Writing region close event to WAL at 1732666553929 (+79 ms)Running coprocessor post-close hooks at 1732666553932 (+3 ms)Closed at 1732666553932 2024-11-27T00:15:53,933 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:53,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=40e749124bfcc05793d8d1161b1bd9fa, regionState=CLOSED 2024-11-27T00:15:53,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:15:53,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-27T00:15:53,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 40e749124bfcc05793d8d1161b1bd9fa, server=f036bc4986ea,33881,1732666453890 in 1.5030 sec 2024-11-27T00:15:53,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-27T00:15:53,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=40e749124bfcc05793d8d1161b1bd9fa, UNASSIGN in 1.5080 sec 2024-11-27T00:15:53,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-27T00:15:53,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5120 sec 2024-11-27T00:15:53,942 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666553942"}]},"ts":"1732666553942"} 2024-11-27T00:15:53,943 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T00:15:53,943 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T00:15:53,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5250 sec 2024-11-27T00:15:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T00:15:54,575 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:54,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T00:15:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:54,579 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T00:15:54,581 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:54,585 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:54,588 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/recovered.edits] 2024-11-27T00:15:54,592 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/1f4f4441043443c5b667ed7ac46d37de to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/1f4f4441043443c5b667ed7ac46d37de 2024-11-27T00:15:54,594 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/5ef2bc6554604256a466fee0534e7561 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/A/5ef2bc6554604256a466fee0534e7561 2024-11-27T00:15:54,597 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/2aadd32855f34bb69e014b76fcffa03b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/2aadd32855f34bb69e014b76fcffa03b 2024-11-27T00:15:54,599 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/538c3528f49649fcb8119bb771141789 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/B/538c3528f49649fcb8119bb771141789 2024-11-27T00:15:54,603 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7433fbabe2704380bc143c552485ddef to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/7433fbabe2704380bc143c552485ddef 2024-11-27T00:15:54,605 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/afda804575a242d0938c54b05cbf3d38 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/C/afda804575a242d0938c54b05cbf3d38 2024-11-27T00:15:54,610 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/recovered.edits/436.seqid to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa/recovered.edits/436.seqid 2024-11-27T00:15:54,610 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/40e749124bfcc05793d8d1161b1bd9fa 2024-11-27T00:15:54,611 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T00:15:54,614 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:54,617 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T00:15:54,620 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T00:15:54,620 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:54,620 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T00:15:54,621 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732666554620"}]},"ts":"9223372036854775807"} 2024-11-27T00:15:54,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T00:15:54,622 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 40e749124bfcc05793d8d1161b1bd9fa, NAME => 'TestAcidGuarantees,,1732666520605.40e749124bfcc05793d8d1161b1bd9fa.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T00:15:54,622 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T00:15:54,623 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732666554622"}]},"ts":"9223372036854775807"} 2024-11-27T00:15:54,624 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T00:15:54,625 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:54,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-27T00:15:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T00:15:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T00:15:54,684 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:54,693 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247 (was 249), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 386) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2940 (was 2274) - AvailableMemoryMB LEAK? - 2024-11-27T00:15:54,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=2940 2024-11-27T00:15:54,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:15:54,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:15:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:54,709 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T00:15:54,709 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:54,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-27T00:15:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T00:15:54,710 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T00:15:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742160_1336 (size=1061) 2024-11-27T00:15:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T00:15:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T00:15:55,120 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 2024-11-27T00:15:55,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742161_1337 (size=53) 2024-11-27T00:15:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T00:15:55,532 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:15:55,532 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 45f2ab312c51b83f191580f6a2814191, disabling compactions & flushes 2024-11-27T00:15:55,532 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:55,532 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:55,532 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. after waiting 0 ms 2024-11-27T00:15:55,532 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:55,532 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:55,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45f2ab312c51b83f191580f6a2814191: Waiting for close lock at 1732666555532Disabling compacts and flushes for region at 1732666555532Disabling writes for close at 1732666555532Writing region close event to WAL at 1732666555532Closed at 1732666555532 2024-11-27T00:15:55,534 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T00:15:55,534 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732666555534"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732666555534"}]},"ts":"1732666555534"} 2024-11-27T00:15:55,535 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T00:15:55,537 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T00:15:55,537 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666555537"}]},"ts":"1732666555537"} 2024-11-27T00:15:55,538 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T00:15:55,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, ASSIGN}] 2024-11-27T00:15:55,540 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, ASSIGN 2024-11-27T00:15:55,540 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, ASSIGN; state=OFFLINE, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=false 2024-11-27T00:15:55,691 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=45f2ab312c51b83f191580f6a2814191, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:55,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, ASSIGN because future has completed 2024-11-27T00:15:55,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:15:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T00:15:55,851 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:55,851 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:15:55,852 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,852 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:15:55,852 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,852 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,854 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,856 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:55,856 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f2ab312c51b83f191580f6a2814191 columnFamilyName A 2024-11-27T00:15:55,856 DEBUG [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:55,857 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(327): Store=45f2ab312c51b83f191580f6a2814191/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:55,857 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,859 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:55,859 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f2ab312c51b83f191580f6a2814191 columnFamilyName B 2024-11-27T00:15:55,859 DEBUG [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:55,860 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(327): Store=45f2ab312c51b83f191580f6a2814191/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:55,860 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,862 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:55,862 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f2ab312c51b83f191580f6a2814191 columnFamilyName C 2024-11-27T00:15:55,862 DEBUG [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:55,863 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(327): Store=45f2ab312c51b83f191580f6a2814191/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:55,863 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:55,863 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,864 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,865 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,866 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,867 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,867 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:15:55,868 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,871 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T00:15:55,871 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 45f2ab312c51b83f191580f6a2814191; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72767801, jitterRate=0.08432473242282867}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:15:55,871 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:55,872 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 45f2ab312c51b83f191580f6a2814191: Running coprocessor pre-open hook at 1732666555852Writing region info on filesystem at 1732666555852Initializing all the Stores at 1732666555853 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666555853Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666555854 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666555854Cleaning up temporary data from old regions at 1732666555867 (+13 ms)Running coprocessor post-open hooks at 1732666555871 (+4 ms)Region opened successfully at 1732666555872 (+1 ms) 2024-11-27T00:15:55,873 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., pid=95, masterSystemTime=1732666555844 2024-11-27T00:15:55,875 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:55,875 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:55,875 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=45f2ab312c51b83f191580f6a2814191, regionState=OPEN, openSeqNum=2, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:55,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:15:55,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-27T00:15:55,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 in 185 msec 2024-11-27T00:15:55,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-27T00:15:55,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, ASSIGN in 341 msec 2024-11-27T00:15:55,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T00:15:55,883 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666555883"}]},"ts":"1732666555883"} 2024-11-27T00:15:55,884 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T00:15:55,885 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T00:15:55,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-27T00:15:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T00:15:56,864 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:56,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:56,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:56,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:56,866 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1f9c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:56,867 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:56,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38443629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:56,868 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:56,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:56,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:56,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:56,870 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T00:15:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f036bc4986ea,33599,1732666453115 2024-11-27T00:15:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53b2e545 2024-11-27T00:15:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T00:15:56,871 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T00:15:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:15:56,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:15:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T00:15:56,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742162_1338 (size=1097) 2024-11-27T00:15:57,285 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-27T00:15:57,285 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-27T00:15:57,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:15:57,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, REOPEN/MOVE}] 2024-11-27T00:15:57,294 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, REOPEN/MOVE 2024-11-27T00:15:57,295 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=45f2ab312c51b83f191580f6a2814191, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:57,296 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33599 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=f036bc4986ea,33881,1732666453890, table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T00:15:57,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, REOPEN/MOVE because future has completed 2024-11-27T00:15:57,296 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:15:57,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:15:57,449 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,449 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:15:57,449 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 45f2ab312c51b83f191580f6a2814191, disabling compactions & flushes 2024-11-27T00:15:57,449 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,449 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,449 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. after waiting 0 ms 2024-11-27T00:15:57,449 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,452 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T00:15:57,453 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,453 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 45f2ab312c51b83f191580f6a2814191: Waiting for close lock at 1732666557449Running coprocessor pre-close hooks at 1732666557449Disabling compacts and flushes for region at 1732666557449Disabling writes for close at 1732666557449Writing region close event to WAL at 1732666557449Running coprocessor post-close hooks at 1732666557453 (+4 ms)Closed at 1732666557453 2024-11-27T00:15:57,453 WARN [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 45f2ab312c51b83f191580f6a2814191 to self. 2024-11-27T00:15:57,455 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,456 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=45f2ab312c51b83f191580f6a2814191, regionState=CLOSED 2024-11-27T00:15:57,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:15:57,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-27T00:15:57,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 in 162 msec 2024-11-27T00:15:57,461 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, REOPEN/MOVE; state=CLOSED, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=true 2024-11-27T00:15:57,612 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=45f2ab312c51b83f191580f6a2814191, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:57,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, REOPEN/MOVE because future has completed 2024-11-27T00:15:57,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:15:57,773 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,773 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:15:57,774 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,774 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:15:57,774 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,774 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,776 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,777 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:57,778 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f2ab312c51b83f191580f6a2814191 columnFamilyName A 2024-11-27T00:15:57,780 DEBUG [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:57,781 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(327): Store=45f2ab312c51b83f191580f6a2814191/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:57,781 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,782 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:57,783 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f2ab312c51b83f191580f6a2814191 columnFamilyName B 2024-11-27T00:15:57,783 DEBUG [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:57,784 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(327): Store=45f2ab312c51b83f191580f6a2814191/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:57,784 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,785 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:15:57,785 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f2ab312c51b83f191580f6a2814191 columnFamilyName C 2024-11-27T00:15:57,785 DEBUG [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:57,785 INFO [StoreOpener-45f2ab312c51b83f191580f6a2814191-1 {}] regionserver.HStore(327): Store=45f2ab312c51b83f191580f6a2814191/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:15:57,786 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,786 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,787 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,788 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,790 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,790 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,790 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:15:57,792 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,792 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 45f2ab312c51b83f191580f6a2814191; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63139848, jitterRate=-0.05914294719696045}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:15:57,792 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,793 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 45f2ab312c51b83f191580f6a2814191: Running coprocessor pre-open hook at 1732666557774Writing region info on filesystem at 1732666557774Initializing all the Stores at 1732666557775 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666557775Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666557776 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666557776Cleaning up temporary data from old regions at 1732666557790 (+14 ms)Running coprocessor post-open hooks at 1732666557793 (+3 ms)Region opened successfully at 1732666557793 2024-11-27T00:15:57,794 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., pid=100, masterSystemTime=1732666557768 2024-11-27T00:15:57,796 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,796 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,797 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=45f2ab312c51b83f191580f6a2814191, regionState=OPEN, openSeqNum=5, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:57,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:15:57,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-27T00:15:57,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 in 184 msec 2024-11-27T00:15:57,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-27T00:15:57,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, REOPEN/MOVE in 507 msec 2024-11-27T00:15:57,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-27T00:15:57,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-11-27T00:15:57,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 933 msec 2024-11-27T00:15:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-27T00:15:57,809 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:15:57,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,811 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ce973f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,812 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4604e01a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,813 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51904f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,814 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef14f81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,815 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7da756, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,816 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb9907b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,818 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1109704c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,819 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c20886c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,820 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe6b0f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb4ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,823 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6787dc9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,824 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e7c30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68269a6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d4b3c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9ebe3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68eeffd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,833 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9d5a54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d270f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:15:57,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:15:57,836 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:15:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:15:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:15:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a528efe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:15:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:15:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:15:57,837 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:15:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700cb5d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:15:57,838 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,838 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,839 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:15:57,839 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,839 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-27T00:15:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,841 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:15:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,841 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,841 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T00:15:57,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,842 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:15:57,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:15:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,842 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,843 DEBUG [Thread-1551 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,843 DEBUG [Thread-1551 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,844 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,845 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:15:57,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:15:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:15:57,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:15:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:15:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:15:57,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:15:57,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:57,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:15:57,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:57,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:15:57,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:15:57,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c384da6828914c5a9bab775b8f07a6b4_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666557855/Put/seqid=0 2024-11-27T00:15:57,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742163_1339 (size=14594) 2024-11-27T00:15:57,891 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:57,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c384da6828914c5a9bab775b8f07a6b4_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c384da6828914c5a9bab775b8f07a6b4_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:57,896 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/471e58b7f73b41fd8f07dbfa431df52d, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:15:57,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/471e58b7f73b41fd8f07dbfa431df52d is 175, key is test_row_0/A:col10/1732666557855/Put/seqid=0 2024-11-27T00:15:57,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742164_1340 (size=39545) 2024-11-27T00:15:57,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666557890, max timestamp=1732666557900Num uniques 859; 2024-11-27T00:15:57,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666557890, max timestamp=1732666557900Num uniques 859; 2024-11-27T00:15:57,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:57,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:57,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666557890, max timestamp=1732666557900Num uniques 859; 2024-11-27T00:15:57,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:15:57,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666617926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:57,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666617927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:57,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666617932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:57,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666617935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T00:15:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:57,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666617936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:57,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:57,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:57,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T00:15:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:15:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:57,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666618044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666618044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666618044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666618054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666618054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T00:15:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:15:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T00:15:58,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666618254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666618254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666618254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666618264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666618264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,301 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/471e58b7f73b41fd8f07dbfa431df52d 2024-11-27T00:15:58,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T00:15:58,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:15:58,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/4b1fb740887f416eabb8abc6c0e21305 is 50, key is test_row_0/B:col10/1732666557855/Put/seqid=0 2024-11-27T00:15:58,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742165_1341 (size=9657) 2024-11-27T00:15:58,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/4b1fb740887f416eabb8abc6c0e21305 2024-11-27T00:15:58,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/3e54e1969d294476a9d6c35761406e56 is 50, key is test_row_0/C:col10/1732666557855/Put/seqid=0 2024-11-27T00:15:58,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742166_1342 (size=9657) 2024-11-27T00:15:58,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T00:15:58,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:15:58,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T00:15:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666618563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666618563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666618574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666618584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666618584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:58,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T00:15:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:15:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T00:15:58,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:15:58,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:15:58,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/3e54e1969d294476a9d6c35761406e56 2024-11-27T00:15:58,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/471e58b7f73b41fd8f07dbfa431df52d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/471e58b7f73b41fd8f07dbfa431df52d 2024-11-27T00:15:58,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/471e58b7f73b41fd8f07dbfa431df52d, entries=200, sequenceid=14, filesize=38.6 K 2024-11-27T00:15:58,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/4b1fb740887f416eabb8abc6c0e21305 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4b1fb740887f416eabb8abc6c0e21305 2024-11-27T00:15:58,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4b1fb740887f416eabb8abc6c0e21305, entries=100, sequenceid=14, filesize=9.4 K 2024-11-27T00:15:58,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/3e54e1969d294476a9d6c35761406e56 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3e54e1969d294476a9d6c35761406e56 2024-11-27T00:15:58,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3e54e1969d294476a9d6c35761406e56, entries=100, sequenceid=14, filesize=9.4 K 2024-11-27T00:15:58,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 45f2ab312c51b83f191580f6a2814191 in 1003ms, sequenceid=14, compaction requested=false 2024-11-27T00:15:58,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:15:58,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T00:15:58,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:15:58,921 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T00:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:15:58,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277bed36343d684265afa94b2569bc0bf3_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666557933/Put/seqid=0 2024-11-27T00:15:58,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742167_1343 (size=12154) 2024-11-27T00:15:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T00:15:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:59,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:15:59,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666619103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666619104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666619104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666619106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666619108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666619214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666619224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666619224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:15:59,339 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277bed36343d684265afa94b2569bc0bf3_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277bed36343d684265afa94b2569bc0bf3_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:15:59,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/c350c1401d8a4ebea0b7febbe70ed050, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:15:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/c350c1401d8a4ebea0b7febbe70ed050 is 175, key is test_row_0/A:col10/1732666557933/Put/seqid=0 2024-11-27T00:15:59,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742168_1344 (size=30955) 2024-11-27T00:15:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666619424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666619443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666619443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,461 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T00:15:59,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666619734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,750 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/c350c1401d8a4ebea0b7febbe70ed050 2024-11-27T00:15:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666619754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/93e50463ce624eeb8be7a1b09d6299eb is 50, key is test_row_0/B:col10/1732666557933/Put/seqid=0 2024-11-27T00:15:59,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742169_1345 (size=12001) 2024-11-27T00:15:59,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:15:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666619764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:15:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:15:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T00:16:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666620124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,130 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666620134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,144 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,167 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/93e50463ce624eeb8be7a1b09d6299eb 2024-11-27T00:16:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/29aea5eafb3340f78473bd30642c3b67 is 50, key is test_row_0/C:col10/1732666557933/Put/seqid=0 2024-11-27T00:16:00,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742170_1346 (size=12001) 2024-11-27T00:16:00,207 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/29aea5eafb3340f78473bd30642c3b67 2024-11-27T00:16:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/c350c1401d8a4ebea0b7febbe70ed050 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/c350c1401d8a4ebea0b7febbe70ed050 2024-11-27T00:16:00,216 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/c350c1401d8a4ebea0b7febbe70ed050, entries=150, sequenceid=50, filesize=30.2 K 2024-11-27T00:16:00,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/93e50463ce624eeb8be7a1b09d6299eb as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/93e50463ce624eeb8be7a1b09d6299eb 2024-11-27T00:16:00,220 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/93e50463ce624eeb8be7a1b09d6299eb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T00:16:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/29aea5eafb3340f78473bd30642c3b67 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/29aea5eafb3340f78473bd30642c3b67 2024-11-27T00:16:00,229 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/29aea5eafb3340f78473bd30642c3b67, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T00:16:00,231 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 45f2ab312c51b83f191580f6a2814191 in 1310ms, sequenceid=50, compaction requested=false 2024-11-27T00:16:00,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:00,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:00,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-27T00:16:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-27T00:16:00,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-27T00:16:00,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3930 sec 2024-11-27T00:16:00,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3980 sec 2024-11-27T00:16:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:00,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T00:16:00,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:00,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:00,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:00,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:00,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:00,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:00,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e227d0455781482f83e3f5b17fbe9e90_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666560247/Put/seqid=0 2024-11-27T00:16:00,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742171_1347 (size=14594) 2024-11-27T00:16:00,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666620308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666620309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666560303, max timestamp=1732666560318Num uniques 859; 2024-11-27T00:16:00,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666560303, max timestamp=1732666560318Num uniques 859; 2024-11-27T00:16:00,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:16:00,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:16:00,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666560303, max timestamp=1732666560318Num uniques 859; 2024-11-27T00:16:00,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:16:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666620434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666620435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666620442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666620554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666620554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666620554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:00,677 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e227d0455781482f83e3f5b17fbe9e90_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e227d0455781482f83e3f5b17fbe9e90_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:00,678 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/618be1258fcb4a1798632dd12bb4daaf, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:00,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/618be1258fcb4a1798632dd12bb4daaf is 175, key is test_row_0/A:col10/1732666560247/Put/seqid=0 2024-11-27T00:16:00,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742172_1348 (size=39549) 2024-11-27T00:16:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666620774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666620774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666620774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:01,082 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/618be1258fcb4a1798632dd12bb4daaf 2024-11-27T00:16:01,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/22a75195e326416a817b21dcea11a88b is 50, key is test_row_0/B:col10/1732666560247/Put/seqid=0 2024-11-27T00:16:01,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742173_1349 (size=12001) 2024-11-27T00:16:01,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666621094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:01,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666621094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:01,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666621094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:01,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/22a75195e326416a817b21dcea11a88b 2024-11-27T00:16:01,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/b29f7bcab369406ba84fdd4d63f2c849 is 50, key is test_row_0/C:col10/1732666560247/Put/seqid=0 2024-11-27T00:16:01,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742174_1350 (size=12001) 2024-11-27T00:16:01,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666621614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:01,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666621614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666621614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:01,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/b29f7bcab369406ba84fdd4d63f2c849 2024-11-27T00:16:01,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/618be1258fcb4a1798632dd12bb4daaf as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/618be1258fcb4a1798632dd12bb4daaf 2024-11-27T00:16:01,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/618be1258fcb4a1798632dd12bb4daaf, entries=200, sequenceid=61, filesize=38.6 K 2024-11-27T00:16:01,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/22a75195e326416a817b21dcea11a88b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/22a75195e326416a817b21dcea11a88b 2024-11-27T00:16:01,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/22a75195e326416a817b21dcea11a88b, entries=150, sequenceid=61, filesize=11.7 K 2024-11-27T00:16:01,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/b29f7bcab369406ba84fdd4d63f2c849 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b29f7bcab369406ba84fdd4d63f2c849 2024-11-27T00:16:01,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b29f7bcab369406ba84fdd4d63f2c849, entries=150, sequenceid=61, filesize=11.7 K 2024-11-27T00:16:01,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 45f2ab312c51b83f191580f6a2814191 in 1682ms, sequenceid=61, compaction requested=true 2024-11-27T00:16:01,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:01,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:01,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:01,930 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:01,931 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:01,931 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:01,931 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:01,931 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/A is initiating minor compaction (all files) 2024-11-27T00:16:01,931 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/B is initiating minor compaction (all files) 2024-11-27T00:16:01,931 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/A in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:01,931 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/B in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:01,932 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4b1fb740887f416eabb8abc6c0e21305, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/93e50463ce624eeb8be7a1b09d6299eb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/22a75195e326416a817b21dcea11a88b] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=32.9 K 2024-11-27T00:16:01,932 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/471e58b7f73b41fd8f07dbfa431df52d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/c350c1401d8a4ebea0b7febbe70ed050, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/618be1258fcb4a1798632dd12bb4daaf] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=107.5 K 2024-11-27T00:16:01,932 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:01,932 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/471e58b7f73b41fd8f07dbfa431df52d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/c350c1401d8a4ebea0b7febbe70ed050, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/618be1258fcb4a1798632dd12bb4daaf] 2024-11-27T00:16:01,932 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1fb740887f416eabb8abc6c0e21305, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732666557849 2024-11-27T00:16:01,932 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 471e58b7f73b41fd8f07dbfa431df52d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732666557849 2024-11-27T00:16:01,933 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e50463ce624eeb8be7a1b09d6299eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732666557922 2024-11-27T00:16:01,933 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a75195e326416a817b21dcea11a88b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732666559092 2024-11-27T00:16:01,940 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:01,940 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/50888595e8864e1aa238700693a10358 is 50, key is test_row_0/B:col10/1732666560247/Put/seqid=0 2024-11-27T00:16:01,940 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c350c1401d8a4ebea0b7febbe70ed050, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732666557922 2024-11-27T00:16:01,941 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 618be1258fcb4a1798632dd12bb4daaf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732666559092 2024-11-27T00:16:01,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742175_1351 (size=12104) 2024-11-27T00:16:01,946 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:01,948 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112788851d97f2ad46e48b8407e215f26a96_45f2ab312c51b83f191580f6a2814191 store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:01,951 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112788851d97f2ad46e48b8407e215f26a96_45f2ab312c51b83f191580f6a2814191, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:01,951 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112788851d97f2ad46e48b8407e215f26a96_45f2ab312c51b83f191580f6a2814191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:01,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742176_1352 (size=4469) 2024-11-27T00:16:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T00:16:02,014 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:02,014 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-27T00:16:02,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T00:16:02,018 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:02,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T00:16:02,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-27T00:16:02,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:02,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:02,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:02,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:02,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:02,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:02,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273fa65fa595b84dd381a3686d5e94efa1_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666560434/Put/seqid=0 2024-11-27T00:16:02,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T00:16:02,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:02,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742177_1353 (size=14594) 2024-11-27T00:16:02,195 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666622192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,200 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273fa65fa595b84dd381a3686d5e94efa1_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273fa65fa595b84dd381a3686d5e94efa1_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:02,202 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/857c53803635450297c20d2780f749fb, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:02,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/857c53803635450297c20d2780f749fb is 175, key is test_row_0/A:col10/1732666560434/Put/seqid=0 2024-11-27T00:16:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666622194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742178_1354 (size=39549) 2024-11-27T00:16:02,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/857c53803635450297c20d2780f749fb 2024-11-27T00:16:02,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/2b1892740136466097e97ef28fbb6339 is 50, key is test_row_0/B:col10/1732666560434/Put/seqid=0 2024-11-27T00:16:02,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742179_1355 (size=12001) 2024-11-27T00:16:02,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666622304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666622314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T00:16:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T00:16:02,354 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/50888595e8864e1aa238700693a10358 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/50888595e8864e1aa238700693a10358 2024-11-27T00:16:02,358 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/B of 45f2ab312c51b83f191580f6a2814191 into 50888595e8864e1aa238700693a10358(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:02,359 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:02,359 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/B, priority=13, startTime=1732666561930; duration=0sec 2024-11-27T00:16:02,359 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:02,359 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:B 2024-11-27T00:16:02,359 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:02,360 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:02,360 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/C is initiating minor compaction (all files) 2024-11-27T00:16:02,360 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/C in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,360 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3e54e1969d294476a9d6c35761406e56, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/29aea5eafb3340f78473bd30642c3b67, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b29f7bcab369406ba84fdd4d63f2c849] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=32.9 K 2024-11-27T00:16:02,360 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e54e1969d294476a9d6c35761406e56, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732666557849 2024-11-27T00:16:02,361 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 29aea5eafb3340f78473bd30642c3b67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732666557922 2024-11-27T00:16:02,361 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b29f7bcab369406ba84fdd4d63f2c849, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732666559092 2024-11-27T00:16:02,366 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:02,367 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/f0f8eecee4194c91a46e06e875cb2a6d is 175, key is test_row_0/A:col10/1732666560247/Put/seqid=0 2024-11-27T00:16:02,393 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#C#compaction#307 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:02,394 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/06d2071d5f91495dbf991a4aa2f5313a is 50, key is test_row_0/C:col10/1732666560247/Put/seqid=0 2024-11-27T00:16:02,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742180_1356 (size=31058) 2024-11-27T00:16:02,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742181_1357 (size=12104) 2024-11-27T00:16:02,446 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/06d2071d5f91495dbf991a4aa2f5313a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/06d2071d5f91495dbf991a4aa2f5313a 2024-11-27T00:16:02,451 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/C of 45f2ab312c51b83f191580f6a2814191 into 06d2071d5f91495dbf991a4aa2f5313a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:02,451 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:02,451 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/C, priority=13, startTime=1732666561931; duration=0sec 2024-11-27T00:16:02,451 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:02,451 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:C 2024-11-27T00:16:02,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T00:16:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666622514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666622534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T00:16:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:02,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666622634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,641 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T00:16:02,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666622644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666622644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:02,656 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,658 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:02,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:02,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:02,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/2b1892740136466097e97ef28fbb6339 2024-11-27T00:16:02,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/e7b5148b9c2e4f5aa35b82964b9fc290 is 50, key is test_row_0/C:col10/1732666560434/Put/seqid=0 2024-11-27T00:16:02,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742182_1358 (size=12001) 2024-11-27T00:16:02,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/e7b5148b9c2e4f5aa35b82964b9fc290 2024-11-27T00:16:02,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/857c53803635450297c20d2780f749fb as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/857c53803635450297c20d2780f749fb 2024-11-27T00:16:02,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/857c53803635450297c20d2780f749fb, entries=200, sequenceid=98, filesize=38.6 K 2024-11-27T00:16:02,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/2b1892740136466097e97ef28fbb6339 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/2b1892740136466097e97ef28fbb6339 2024-11-27T00:16:02,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/2b1892740136466097e97ef28fbb6339, entries=150, sequenceid=98, filesize=11.7 K 2024-11-27T00:16:02,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/e7b5148b9c2e4f5aa35b82964b9fc290 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/e7b5148b9c2e4f5aa35b82964b9fc290 2024-11-27T00:16:02,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/e7b5148b9c2e4f5aa35b82964b9fc290, entries=150, sequenceid=98, filesize=11.7 K 2024-11-27T00:16:02,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 45f2ab312c51b83f191580f6a2814191 in 630ms, sequenceid=98, compaction requested=false 2024-11-27T00:16:02,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:02,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T00:16:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:02,791 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:16:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:02,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:02,815 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/f0f8eecee4194c91a46e06e875cb2a6d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/f0f8eecee4194c91a46e06e875cb2a6d 2024-11-27T00:16:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112703b48658b6d447a094d76a0159c4c85c_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666562159/Put/seqid=0 2024-11-27T00:16:02,844 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/A of 45f2ab312c51b83f191580f6a2814191 into f0f8eecee4194c91a46e06e875cb2a6d(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:02,844 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:02,844 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/A, priority=13, startTime=1732666561930; duration=0sec 2024-11-27T00:16:02,844 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:02,844 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:A 2024-11-27T00:16:02,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:02,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742183_1359 (size=12154) 2024-11-27T00:16:02,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:02,891 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112703b48658b6d447a094d76a0159c4c85c_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112703b48658b6d447a094d76a0159c4c85c_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:02,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/96fb37ab02f149f2985352e45ab78f05, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/96fb37ab02f149f2985352e45ab78f05 is 175, key is test_row_0/A:col10/1732666562159/Put/seqid=0 2024-11-27T00:16:02,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742184_1360 (size=30955) 2024-11-27T00:16:02,959 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/96fb37ab02f149f2985352e45ab78f05 2024-11-27T00:16:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666562911, max timestamp=1732666562967Num uniques 859; 2024-11-27T00:16:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:16:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666562911, max timestamp=1732666562968Num uniques 859; 2024-11-27T00:16:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732666562911, max timestamp=1732666562968Num uniques 859; 2024-11-27T00:16:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:16:02,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-27T00:16:02,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/e9d599f4a2dc4484abcb4fd0bf5841ba is 50, key is test_row_0/B:col10/1732666562159/Put/seqid=0 2024-11-27T00:16:03,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742185_1361 (size=12001) 2024-11-27T00:16:03,019 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/e9d599f4a2dc4484abcb4fd0bf5841ba 2024-11-27T00:16:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/bc303de9317a43149a22b75694e6beb4 is 50, key is test_row_0/C:col10/1732666562159/Put/seqid=0 2024-11-27T00:16:03,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742186_1362 (size=12001) 2024-11-27T00:16:03,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666623071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666623082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T00:16:03,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666623194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666623205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666623414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666623424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,454 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/bc303de9317a43149a22b75694e6beb4 2024-11-27T00:16:03,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/96fb37ab02f149f2985352e45ab78f05 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/96fb37ab02f149f2985352e45ab78f05 2024-11-27T00:16:03,471 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/96fb37ab02f149f2985352e45ab78f05, entries=150, sequenceid=110, filesize=30.2 K 2024-11-27T00:16:03,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/e9d599f4a2dc4484abcb4fd0bf5841ba as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e9d599f4a2dc4484abcb4fd0bf5841ba 2024-11-27T00:16:03,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e9d599f4a2dc4484abcb4fd0bf5841ba, entries=150, sequenceid=110, filesize=11.7 K 2024-11-27T00:16:03,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/bc303de9317a43149a22b75694e6beb4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/bc303de9317a43149a22b75694e6beb4 2024-11-27T00:16:03,484 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/bc303de9317a43149a22b75694e6beb4, entries=150, sequenceid=110, filesize=11.7 K 2024-11-27T00:16:03,489 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 45f2ab312c51b83f191580f6a2814191 in 698ms, sequenceid=110, compaction requested=true 2024-11-27T00:16:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-27T00:16:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-27T00:16:03,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-27T00:16:03,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-11-27T00:16:03,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4810 sec 2024-11-27T00:16:03,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732666563030, max timestamp=1732666563740Num uniques 1018; 2024-11-27T00:16:03,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-27T00:16:03,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=102.87 KB heapSize=292.20 KB 2024-11-27T00:16:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T00:16:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-27T00:16:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:03,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274cf99cbf350d4f4a823bd6da3f14a76f_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666563066/Put/seqid=0 2024-11-27T00:16:03,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742187_1363 (size=14794) 2024-11-27T00:16:03,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666623816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666623831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666623934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:03,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666623944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666624143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666624164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T00:16:04,174 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:04,175 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-27T00:16:04,176 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274cf99cbf350d4f4a823bd6da3f14a76f_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274cf99cbf350d4f4a823bd6da3f14a76f_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:04,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T00:16:04,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:04,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:04,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8b763157f7e94bd7b497c77b9c4c1ab5, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:04,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8b763157f7e94bd7b497c77b9c4c1ab5 is 175, key is test_row_0/A:col10/1732666563066/Put/seqid=0 2024-11-27T00:16:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742188_1364 (size=39749) 2024-11-27T00:16:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T00:16:04,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666624463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666624474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:04,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T00:16:04,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8b763157f7e94bd7b497c77b9c4c1ab5 2024-11-27T00:16:04,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/d47c5f778b5b4a40887b0799fc29b90d is 50, key is test_row_0/B:col10/1732666563066/Put/seqid=0 2024-11-27T00:16:04,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742189_1365 (size=12151) 2024-11-27T00:16:04,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:04,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:04,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666624664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666624664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:04,673 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666624664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,673 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,673 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T00:16:04,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:04,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:04,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:04,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666624984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:04,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/d47c5f778b5b4a40887b0799fc29b90d 2024-11-27T00:16:05,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/51a503a962714fd18cebcc9ed479f6df is 50, key is test_row_0/C:col10/1732666563066/Put/seqid=0 2024-11-27T00:16:05,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742190_1366 (size=12151) 2024-11-27T00:16:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666624994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:05,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:05,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:05,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T00:16:05,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:05,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:05,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:05,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/51a503a962714fd18cebcc9ed479f6df 2024-11-27T00:16:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8b763157f7e94bd7b497c77b9c4c1ab5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8b763157f7e94bd7b497c77b9c4c1ab5 2024-11-27T00:16:05,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8b763157f7e94bd7b497c77b9c4c1ab5, entries=200, sequenceid=149, filesize=38.8 K 2024-11-27T00:16:05,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/d47c5f778b5b4a40887b0799fc29b90d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/d47c5f778b5b4a40887b0799fc29b90d 2024-11-27T00:16:05,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/d47c5f778b5b4a40887b0799fc29b90d, entries=150, sequenceid=149, filesize=11.9 K 2024-11-27T00:16:05,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/51a503a962714fd18cebcc9ed479f6df as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/51a503a962714fd18cebcc9ed479f6df 2024-11-27T00:16:05,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/51a503a962714fd18cebcc9ed479f6df, entries=150, sequenceid=149, filesize=11.9 K 2024-11-27T00:16:05,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~292.16 KB/299168, currentSize=73.80 KB/75570 for 45f2ab312c51b83f191580f6a2814191 in 1679ms, sequenceid=149, compaction requested=true 2024-11-27T00:16:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:05,424 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:05,424 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:05,425 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:05,425 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:05,425 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/B is initiating minor compaction (all files) 2024-11-27T00:16:05,425 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/A is initiating minor compaction (all files) 2024-11-27T00:16:05,425 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/B in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,425 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/A in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,426 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/50888595e8864e1aa238700693a10358, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/2b1892740136466097e97ef28fbb6339, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e9d599f4a2dc4484abcb4fd0bf5841ba, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/d47c5f778b5b4a40887b0799fc29b90d] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=47.1 K 2024-11-27T00:16:05,426 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/f0f8eecee4194c91a46e06e875cb2a6d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/857c53803635450297c20d2780f749fb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/96fb37ab02f149f2985352e45ab78f05, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8b763157f7e94bd7b497c77b9c4c1ab5] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=138.0 K 2024-11-27T00:16:05,426 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,426 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/f0f8eecee4194c91a46e06e875cb2a6d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/857c53803635450297c20d2780f749fb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/96fb37ab02f149f2985352e45ab78f05, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8b763157f7e94bd7b497c77b9c4c1ab5] 2024-11-27T00:16:05,426 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 50888595e8864e1aa238700693a10358, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732666559092 2024-11-27T00:16:05,426 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f8eecee4194c91a46e06e875cb2a6d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732666559092 2024-11-27T00:16:05,426 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b1892740136466097e97ef28fbb6339, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732666560434 2024-11-27T00:16:05,426 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d599f4a2dc4484abcb4fd0bf5841ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732666562159 2024-11-27T00:16:05,426 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 857c53803635450297c20d2780f749fb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732666560425 2024-11-27T00:16:05,426 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d47c5f778b5b4a40887b0799fc29b90d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732666563066 2024-11-27T00:16:05,426 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96fb37ab02f149f2985352e45ab78f05, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732666562159 2024-11-27T00:16:05,427 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b763157f7e94bd7b497c77b9c4c1ab5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732666563030 2024-11-27T00:16:05,437 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:05,437 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/0d6e744bc21649aa8485632281f69ef5 is 50, key is test_row_0/B:col10/1732666563066/Put/seqid=0 2024-11-27T00:16:05,444 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:05,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742191_1367 (size=12391) 2024-11-27T00:16:05,446 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411278af688e9c0494a09a99549238cf56354_45f2ab312c51b83f191580f6a2814191 store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:05,453 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411278af688e9c0494a09a99549238cf56354_45f2ab312c51b83f191580f6a2814191, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:05,454 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278af688e9c0494a09a99549238cf56354_45f2ab312c51b83f191580f6a2814191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742192_1368 (size=4469) 2024-11-27T00:16:05,460 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#A#compaction#316 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:05,461 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/df4e9482e7ab44e6be9df3899df2e4a6 is 175, key is test_row_0/A:col10/1732666563066/Put/seqid=0 2024-11-27T00:16:05,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742193_1369 (size=31345) 2024-11-27T00:16:05,475 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/df4e9482e7ab44e6be9df3899df2e4a6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/df4e9482e7ab44e6be9df3899df2e4a6 2024-11-27T00:16:05,482 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/A of 45f2ab312c51b83f191580f6a2814191 into df4e9482e7ab44e6be9df3899df2e4a6(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:05,482 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:05,482 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/A, priority=12, startTime=1732666565424; duration=0sec 2024-11-27T00:16:05,482 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:05,482 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:A 2024-11-27T00:16:05,482 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:05,483 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:05,483 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/C is initiating minor compaction (all files) 2024-11-27T00:16:05,483 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/C in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,483 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/06d2071d5f91495dbf991a4aa2f5313a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/e7b5148b9c2e4f5aa35b82964b9fc290, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/bc303de9317a43149a22b75694e6beb4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/51a503a962714fd18cebcc9ed479f6df] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=47.1 K 2024-11-27T00:16:05,483 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d2071d5f91495dbf991a4aa2f5313a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732666559092 2024-11-27T00:16:05,483 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7b5148b9c2e4f5aa35b82964b9fc290, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732666560434 2024-11-27T00:16:05,484 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc303de9317a43149a22b75694e6beb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732666562159 2024-11-27T00:16:05,484 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a503a962714fd18cebcc9ed479f6df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732666563066 2024-11-27T00:16:05,497 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#C#compaction#317 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:05,497 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/a16b3960f72a4e6bbf06e3c05655cf92 is 50, key is test_row_0/C:col10/1732666563066/Put/seqid=0 2024-11-27T00:16:05,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742194_1370 (size=12391) 2024-11-27T00:16:05,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T00:16:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:05,552 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:16:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:05,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:05,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276ac5d5d17454404e95419440c466b679_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666563812/Put/seqid=0 2024-11-27T00:16:05,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742195_1371 (size=12304) 2024-11-27T00:16:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:05,565 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276ac5d5d17454404e95419440c466b679_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276ac5d5d17454404e95419440c466b679_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8de0a0b7ac274770a620cbdaeca1b0f4, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:05,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8de0a0b7ac274770a620cbdaeca1b0f4 is 175, key is test_row_0/A:col10/1732666563812/Put/seqid=0 2024-11-27T00:16:05,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742196_1372 (size=31105) 2024-11-27T00:16:05,849 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/0d6e744bc21649aa8485632281f69ef5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0d6e744bc21649aa8485632281f69ef5 2024-11-27T00:16:05,853 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/B of 45f2ab312c51b83f191580f6a2814191 into 0d6e744bc21649aa8485632281f69ef5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:05,853 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:05,853 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/B, priority=12, startTime=1732666565424; duration=0sec 2024-11-27T00:16:05,853 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:05,853 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:B 2024-11-27T00:16:05,920 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/a16b3960f72a4e6bbf06e3c05655cf92 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a16b3960f72a4e6bbf06e3c05655cf92 2024-11-27T00:16:05,926 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/C of 45f2ab312c51b83f191580f6a2814191 into a16b3960f72a4e6bbf06e3c05655cf92(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:05,926 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:05,926 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/C, priority=12, startTime=1732666565424; duration=0sec 2024-11-27T00:16:05,926 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:05,926 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:C 2024-11-27T00:16:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:06,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:06,022 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8de0a0b7ac274770a620cbdaeca1b0f4 2024-11-27T00:16:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/0c4eb457ca3141c9827074e302838aff is 50, key is test_row_0/B:col10/1732666563812/Put/seqid=0 2024-11-27T00:16:06,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742197_1373 (size=12151) 2024-11-27T00:16:06,042 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/0c4eb457ca3141c9827074e302838aff 2024-11-27T00:16:06,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/82f96bc17838497ebc036f8a3a472720 is 50, key is test_row_0/C:col10/1732666563812/Put/seqid=0 2024-11-27T00:16:06,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742198_1374 (size=12151) 2024-11-27T00:16:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666626077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666626078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666626194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666626194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T00:16:06,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666626404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666626414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,453 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/82f96bc17838497ebc036f8a3a472720 2024-11-27T00:16:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8de0a0b7ac274770a620cbdaeca1b0f4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8de0a0b7ac274770a620cbdaeca1b0f4 2024-11-27T00:16:06,461 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8de0a0b7ac274770a620cbdaeca1b0f4, entries=150, sequenceid=164, filesize=30.4 K 2024-11-27T00:16:06,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/0c4eb457ca3141c9827074e302838aff as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0c4eb457ca3141c9827074e302838aff 2024-11-27T00:16:06,465 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0c4eb457ca3141c9827074e302838aff, entries=150, sequenceid=164, filesize=11.9 K 2024-11-27T00:16:06,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/82f96bc17838497ebc036f8a3a472720 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/82f96bc17838497ebc036f8a3a472720 2024-11-27T00:16:06,469 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/82f96bc17838497ebc036f8a3a472720, entries=150, sequenceid=164, filesize=11.9 K 2024-11-27T00:16:06,469 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 45f2ab312c51b83f191580f6a2814191 in 917ms, sequenceid=164, compaction requested=false 2024-11-27T00:16:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-27T00:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-27T00:16:06,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-27T00:16:06,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2940 sec 2024-11-27T00:16:06,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.2980 sec 2024-11-27T00:16:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:06,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:16:06,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:06,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:06,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:06,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:06,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:06,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:06,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271d7331a8a3554ce880b18c859be42b7f_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666566077/Put/seqid=0 2024-11-27T00:16:06,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742199_1375 (size=14794) 2024-11-27T00:16:06,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666626762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666626766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666626874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666626874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:07,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666627083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:07,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666627094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:07,141 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:07,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271d7331a8a3554ce880b18c859be42b7f_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271d7331a8a3554ce880b18c859be42b7f_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:07,145 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/21e3d944203e44b4985156d89c0ce9ba, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:07,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/21e3d944203e44b4985156d89c0ce9ba is 175, key is test_row_0/A:col10/1732666566077/Put/seqid=0 2024-11-27T00:16:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742200_1376 (size=39749) 2024-11-27T00:16:07,151 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/21e3d944203e44b4985156d89c0ce9ba 2024-11-27T00:16:07,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/24c1a29b25c54cf3bb8ccdf098876834 is 50, key is test_row_0/B:col10/1732666566077/Put/seqid=0 2024-11-27T00:16:07,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742201_1377 (size=12151) 2024-11-27T00:16:07,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/24c1a29b25c54cf3bb8ccdf098876834 2024-11-27T00:16:07,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/b60f05ebdd4741da975b1c3011c1d1a5 is 50, key is test_row_0/C:col10/1732666566077/Put/seqid=0 2024-11-27T00:16:07,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742202_1378 (size=12151) 2024-11-27T00:16:07,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666627403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:07,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:07,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:07,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:07,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666627414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:07,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/b60f05ebdd4741da975b1c3011c1d1a5 2024-11-27T00:16:07,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/21e3d944203e44b4985156d89c0ce9ba as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/21e3d944203e44b4985156d89c0ce9ba 2024-11-27T00:16:07,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/21e3d944203e44b4985156d89c0ce9ba, entries=200, sequenceid=185, filesize=38.8 K 2024-11-27T00:16:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/24c1a29b25c54cf3bb8ccdf098876834 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/24c1a29b25c54cf3bb8ccdf098876834 2024-11-27T00:16:07,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/24c1a29b25c54cf3bb8ccdf098876834, entries=150, sequenceid=185, filesize=11.9 K 2024-11-27T00:16:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/b60f05ebdd4741da975b1c3011c1d1a5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b60f05ebdd4741da975b1c3011c1d1a5 2024-11-27T00:16:07,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b60f05ebdd4741da975b1c3011c1d1a5, entries=150, sequenceid=185, filesize=11.9 K 2024-11-27T00:16:07,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 45f2ab312c51b83f191580f6a2814191 in 873ms, sequenceid=185, compaction requested=true 2024-11-27T00:16:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:07,601 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:07,601 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:07,605 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:07,605 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:07,605 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/B is initiating minor compaction (all files) 2024-11-27T00:16:07,605 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/A is initiating minor compaction (all files) 2024-11-27T00:16:07,605 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/B in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:07,605 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/A in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:07,605 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0d6e744bc21649aa8485632281f69ef5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0c4eb457ca3141c9827074e302838aff, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/24c1a29b25c54cf3bb8ccdf098876834] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=35.8 K 2024-11-27T00:16:07,605 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/df4e9482e7ab44e6be9df3899df2e4a6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8de0a0b7ac274770a620cbdaeca1b0f4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/21e3d944203e44b4985156d89c0ce9ba] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=99.8 K 2024-11-27T00:16:07,605 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:07,605 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/df4e9482e7ab44e6be9df3899df2e4a6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8de0a0b7ac274770a620cbdaeca1b0f4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/21e3d944203e44b4985156d89c0ce9ba] 2024-11-27T00:16:07,606 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d6e744bc21649aa8485632281f69ef5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732666563066 2024-11-27T00:16:07,606 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting df4e9482e7ab44e6be9df3899df2e4a6, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732666563066 2024-11-27T00:16:07,606 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8de0a0b7ac274770a620cbdaeca1b0f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732666563794 2024-11-27T00:16:07,606 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c4eb457ca3141c9827074e302838aff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732666563794 2024-11-27T00:16:07,606 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 24c1a29b25c54cf3bb8ccdf098876834, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732666566054 2024-11-27T00:16:07,606 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e3d944203e44b4985156d89c0ce9ba, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732666566054 2024-11-27T00:16:07,611 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:07,613 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:07,613 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/495fe36c9786485886800c01d82f9fa6 is 50, key is test_row_0/B:col10/1732666566077/Put/seqid=0 2024-11-27T00:16:07,617 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127447289abf248421ca921d9a4e911821e_45f2ab312c51b83f191580f6a2814191 store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:07,619 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127447289abf248421ca921d9a4e911821e_45f2ab312c51b83f191580f6a2814191, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:07,619 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127447289abf248421ca921d9a4e911821e_45f2ab312c51b83f191580f6a2814191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:07,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742203_1379 (size=12493) 2024-11-27T00:16:07,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742204_1380 (size=4469) 2024-11-27T00:16:07,636 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#A#compaction#325 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:07,636 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/0a0760271b184537bbe92aa4e1903017 is 175, key is test_row_0/A:col10/1732666566077/Put/seqid=0 2024-11-27T00:16:07,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742205_1381 (size=31447) 2024-11-27T00:16:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:07,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:16:07,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:07,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:07,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:07,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:07,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:07,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:07,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112788221186d4c64ee5a0b6d6b7ca0e7d3a_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666566761/Put/seqid=0 2024-11-27T00:16:07,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742206_1382 (size=14794) 2024-11-27T00:16:08,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666628006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666628016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,035 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/495fe36c9786485886800c01d82f9fa6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/495fe36c9786485886800c01d82f9fa6 2024-11-27T00:16:08,043 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/B of 45f2ab312c51b83f191580f6a2814191 into 495fe36c9786485886800c01d82f9fa6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:08,043 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:08,044 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/B, priority=13, startTime=1732666567601; duration=0sec 2024-11-27T00:16:08,044 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:08,044 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:B 2024-11-27T00:16:08,045 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:08,051 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:08,051 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/0a0760271b184537bbe92aa4e1903017 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0a0760271b184537bbe92aa4e1903017 2024-11-27T00:16:08,051 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/C is initiating minor compaction (all files) 2024-11-27T00:16:08,051 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/C in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,051 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a16b3960f72a4e6bbf06e3c05655cf92, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/82f96bc17838497ebc036f8a3a472720, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b60f05ebdd4741da975b1c3011c1d1a5] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=35.8 K 2024-11-27T00:16:08,052 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting a16b3960f72a4e6bbf06e3c05655cf92, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732666563066 2024-11-27T00:16:08,052 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f96bc17838497ebc036f8a3a472720, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732666563794 2024-11-27T00:16:08,052 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b60f05ebdd4741da975b1c3011c1d1a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732666566054 2024-11-27T00:16:08,057 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/A of 45f2ab312c51b83f191580f6a2814191 into 0a0760271b184537bbe92aa4e1903017(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:08,057 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:08,057 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/A, priority=13, startTime=1732666567601; duration=0sec 2024-11-27T00:16:08,057 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:08,057 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:A 2024-11-27T00:16:08,067 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:08,067 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/eac398e47f0a48e7951991f3a505c960 is 50, key is test_row_0/C:col10/1732666566077/Put/seqid=0 2024-11-27T00:16:08,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742207_1383 (size=12493) 2024-11-27T00:16:08,092 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/eac398e47f0a48e7951991f3a505c960 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/eac398e47f0a48e7951991f3a505c960 2024-11-27T00:16:08,098 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/C of 45f2ab312c51b83f191580f6a2814191 into eac398e47f0a48e7951991f3a505c960(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:08,098 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:08,098 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/C, priority=13, startTime=1732666567601; duration=0sec 2024-11-27T00:16:08,098 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:08,098 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:C 2024-11-27T00:16:08,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666628124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666628124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666628334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,343 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T00:16:08,344 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:08,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-27T00:16:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T00:16:08,350 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112788221186d4c64ee5a0b6d6b7ca0e7d3a_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112788221186d4c64ee5a0b6d6b7ca0e7d3a_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:08,350 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:08,351 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:08,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:08,351 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/40bba7711b8c4553a28055c202f7de37, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:08,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/40bba7711b8c4553a28055c202f7de37 is 175, key is test_row_0/A:col10/1732666566761/Put/seqid=0 2024-11-27T00:16:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666628344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742208_1384 (size=39749) 2024-11-27T00:16:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T00:16:08,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:08,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:08,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666628644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T00:16:08,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666628664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666628684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,689 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666628704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,714 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666628715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:08,720 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:08,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:08,770 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/40bba7711b8c4553a28055c202f7de37 2024-11-27T00:16:08,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/4e941b92a7cb4ac9b8c382442677c146 is 50, key is test_row_0/B:col10/1732666566761/Put/seqid=0 2024-11-27T00:16:08,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742209_1385 (size=12151) 2024-11-27T00:16:08,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:08,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:08,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:08,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:08,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:08,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T00:16:09,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:09,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:09,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666629154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:09,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/4e941b92a7cb4ac9b8c382442677c146 2024-11-27T00:16:09,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666629184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:09,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/fb4928c0671845a38300cee85dca1f36 is 50, key is test_row_0/C:col10/1732666566761/Put/seqid=0 2024-11-27T00:16:09,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742210_1386 (size=12151) 2024-11-27T00:16:09,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:09,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:09,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:09,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:09,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T00:16:09,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:09,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:09,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:09,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/fb4928c0671845a38300cee85dca1f36 2024-11-27T00:16:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/40bba7711b8c4553a28055c202f7de37 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/40bba7711b8c4553a28055c202f7de37 2024-11-27T00:16:09,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/40bba7711b8c4553a28055c202f7de37, entries=200, sequenceid=199, filesize=38.8 K 2024-11-27T00:16:09,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/4e941b92a7cb4ac9b8c382442677c146 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4e941b92a7cb4ac9b8c382442677c146 2024-11-27T00:16:09,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4e941b92a7cb4ac9b8c382442677c146, entries=150, sequenceid=199, filesize=11.9 K 2024-11-27T00:16:09,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/fb4928c0671845a38300cee85dca1f36 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/fb4928c0671845a38300cee85dca1f36 2024-11-27T00:16:09,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/fb4928c0671845a38300cee85dca1f36, entries=150, sequenceid=199, filesize=11.9 K 2024-11-27T00:16:09,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 45f2ab312c51b83f191580f6a2814191 in 1699ms, sequenceid=199, compaction requested=false 2024-11-27T00:16:09,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:09,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T00:16:09,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:09,736 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:16:09,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:09,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bcdc487937e642edae85294df3e15432_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666568004/Put/seqid=0 2024-11-27T00:16:09,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742211_1387 (size=12304) 2024-11-27T00:16:10,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:10,154 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bcdc487937e642edae85294df3e15432_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bcdc487937e642edae85294df3e15432_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:10,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/926d10436482481584ffb631f3ebe675, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:10,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/926d10436482481584ffb631f3ebe675 is 175, key is test_row_0/A:col10/1732666568004/Put/seqid=0 2024-11-27T00:16:10,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742212_1388 (size=31105) 2024-11-27T00:16:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:10,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666630220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666630221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:10,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666630333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:10,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666630344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T00:16:10,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666630544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:10,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:10,568 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/926d10436482481584ffb631f3ebe675 2024-11-27T00:16:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666630564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/ab9c4f4a512c451d83c235f413960511 is 50, key is test_row_0/B:col10/1732666568004/Put/seqid=0 2024-11-27T00:16:10,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742213_1389 (size=12151) 2024-11-27T00:16:10,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666630865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:10,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666630884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:11,041 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/ab9c4f4a512c451d83c235f413960511 2024-11-27T00:16:11,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/3ea4b76b14814f63a4b823b2372ee11a is 50, key is test_row_0/C:col10/1732666568004/Put/seqid=0 2024-11-27T00:16:11,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742214_1390 (size=12151) 2024-11-27T00:16:11,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666631385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:11,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666631394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:11,465 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/3ea4b76b14814f63a4b823b2372ee11a 2024-11-27T00:16:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/926d10436482481584ffb631f3ebe675 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/926d10436482481584ffb631f3ebe675 2024-11-27T00:16:11,480 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/926d10436482481584ffb631f3ebe675, entries=150, sequenceid=220, filesize=30.4 K 2024-11-27T00:16:11,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/ab9c4f4a512c451d83c235f413960511 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ab9c4f4a512c451d83c235f413960511 2024-11-27T00:16:11,497 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ab9c4f4a512c451d83c235f413960511, entries=150, sequenceid=220, filesize=11.9 K 2024-11-27T00:16:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/3ea4b76b14814f63a4b823b2372ee11a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3ea4b76b14814f63a4b823b2372ee11a 2024-11-27T00:16:11,506 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3ea4b76b14814f63a4b823b2372ee11a, entries=150, sequenceid=220, filesize=11.9 K 2024-11-27T00:16:11,507 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 45f2ab312c51b83f191580f6a2814191 in 1771ms, sequenceid=220, compaction requested=true 2024-11-27T00:16:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:11,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:11,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-27T00:16:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-27T00:16:11,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-27T00:16:11,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1580 sec 2024-11-27T00:16:11,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.1670 sec 2024-11-27T00:16:12,121 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T00:16:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:12,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:16:12,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:12,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:12,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:12,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:12,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:12,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:12,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fa8a35aaf150489aa15a6bb333a5e8b4_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666570220/Put/seqid=0 2024-11-27T00:16:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742215_1391 (size=14794) 2024-11-27T00:16:12,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666632471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:12,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666632474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T00:16:12,514 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:12,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-27T00:16:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T00:16:12,515 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:12,516 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:12,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:12,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666632594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:12,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666632594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T00:16:12,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T00:16:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:12,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:12,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T00:16:12,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:12,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:12,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:12,821 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:12,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:12,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:12,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666632813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:12,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666632814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:12,825 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fa8a35aaf150489aa15a6bb333a5e8b4_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa8a35aaf150489aa15a6bb333a5e8b4_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:12,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/bb299df137f343cdba8413b68a0cd18c, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:12,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/bb299df137f343cdba8413b68a0cd18c is 175, key is test_row_0/A:col10/1732666570220/Put/seqid=0 2024-11-27T00:16:12,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742216_1392 (size=39749) 2024-11-27T00:16:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T00:16:12,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T00:16:12,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:12,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:12,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:12,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:12,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T00:16:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T00:16:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666633134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:13,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666633134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:13,229 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/bb299df137f343cdba8413b68a0cd18c 2024-11-27T00:16:13,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/cc23ff43ecaa482aa5554eccb5227fbf is 50, key is test_row_0/B:col10/1732666570220/Put/seqid=0 2024-11-27T00:16:13,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742217_1393 (size=12151) 2024-11-27T00:16:13,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/cc23ff43ecaa482aa5554eccb5227fbf 2024-11-27T00:16:13,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/36448fd7730649eb8c9a4d00fb105e8e is 50, key is test_row_0/C:col10/1732666570220/Put/seqid=0 2024-11-27T00:16:13,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742218_1394 (size=12151) 2024-11-27T00:16:13,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T00:16:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T00:16:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T00:16:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T00:16:13,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666633654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:13,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666633655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:13,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/36448fd7730649eb8c9a4d00fb105e8e 2024-11-27T00:16:13,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/bb299df137f343cdba8413b68a0cd18c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/bb299df137f343cdba8413b68a0cd18c 2024-11-27T00:16:13,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/bb299df137f343cdba8413b68a0cd18c, entries=200, sequenceid=235, filesize=38.8 K 2024-11-27T00:16:13,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/cc23ff43ecaa482aa5554eccb5227fbf as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/cc23ff43ecaa482aa5554eccb5227fbf 2024-11-27T00:16:13,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/cc23ff43ecaa482aa5554eccb5227fbf, entries=150, sequenceid=235, filesize=11.9 K 2024-11-27T00:16:13,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/36448fd7730649eb8c9a4d00fb105e8e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/36448fd7730649eb8c9a4d00fb105e8e 2024-11-27T00:16:13,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/36448fd7730649eb8c9a4d00fb105e8e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-27T00:16:13,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 45f2ab312c51b83f191580f6a2814191 in 1267ms, sequenceid=235, compaction requested=true 2024-11-27T00:16:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:13,677 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:13,677 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:13,680 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:13,680 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/B is initiating minor compaction (all files) 2024-11-27T00:16:13,680 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/B in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,680 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/495fe36c9786485886800c01d82f9fa6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4e941b92a7cb4ac9b8c382442677c146, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ab9c4f4a512c451d83c235f413960511, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/cc23ff43ecaa482aa5554eccb5227fbf] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=47.8 K 2024-11-27T00:16:13,681 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:13,681 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/A is initiating minor compaction (all files) 2024-11-27T00:16:13,681 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 495fe36c9786485886800c01d82f9fa6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732666566054 2024-11-27T00:16:13,681 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/A in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,681 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0a0760271b184537bbe92aa4e1903017, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/40bba7711b8c4553a28055c202f7de37, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/926d10436482481584ffb631f3ebe675, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/bb299df137f343cdba8413b68a0cd18c] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=138.7 K 2024-11-27T00:16:13,681 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,681 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e941b92a7cb4ac9b8c382442677c146, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732666566761 2024-11-27T00:16:13,681 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0a0760271b184537bbe92aa4e1903017, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/40bba7711b8c4553a28055c202f7de37, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/926d10436482481584ffb631f3ebe675, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/bb299df137f343cdba8413b68a0cd18c] 2024-11-27T00:16:13,681 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ab9c4f4a512c451d83c235f413960511, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732666568000 2024-11-27T00:16:13,681 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting cc23ff43ecaa482aa5554eccb5227fbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666570206 2024-11-27T00:16:13,683 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a0760271b184537bbe92aa4e1903017, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732666566054 2024-11-27T00:16:13,684 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40bba7711b8c4553a28055c202f7de37, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732666566751 2024-11-27T00:16:13,684 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 926d10436482481584ffb631f3ebe675, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732666568000 2024-11-27T00:16:13,685 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb299df137f343cdba8413b68a0cd18c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666570206 2024-11-27T00:16:13,690 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:13,690 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/622c4ba607414260af28fae4d715f23a is 50, key is test_row_0/B:col10/1732666570220/Put/seqid=0 2024-11-27T00:16:13,690 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:13,692 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127ec64a9b26c9d45d3b9a26cfefb3e3b06_45f2ab312c51b83f191580f6a2814191 store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:13,695 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127ec64a9b26c9d45d3b9a26cfefb3e3b06_45f2ab312c51b83f191580f6a2814191, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:13,695 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ec64a9b26c9d45d3b9a26cfefb3e3b06_45f2ab312c51b83f191580f6a2814191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:13,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742219_1395 (size=4469) 2024-11-27T00:16:13,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742220_1396 (size=12629) 2024-11-27T00:16:13,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T00:16:13,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:13,740 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:16:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:13,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:13,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cc3cf87aa5e34e598e03e20fdce2778f_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666572470/Put/seqid=0 2024-11-27T00:16:13,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742221_1397 (size=12304) 2024-11-27T00:16:14,104 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:14,105 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8d644a78f6664da28ea8f854f3db9994 is 175, key is test_row_0/A:col10/1732666570220/Put/seqid=0 2024-11-27T00:16:14,110 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/622c4ba607414260af28fae4d715f23a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/622c4ba607414260af28fae4d715f23a 2024-11-27T00:16:14,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742222_1398 (size=31583) 2024-11-27T00:16:14,125 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/B of 45f2ab312c51b83f191580f6a2814191 into 622c4ba607414260af28fae4d715f23a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:14,125 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:14,126 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/B, priority=12, startTime=1732666573677; duration=0sec 2024-11-27T00:16:14,126 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:14,126 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:B 2024-11-27T00:16:14,126 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:14,128 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:14,128 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/C is initiating minor compaction (all files) 2024-11-27T00:16:14,128 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/C in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:14,128 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/eac398e47f0a48e7951991f3a505c960, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/fb4928c0671845a38300cee85dca1f36, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3ea4b76b14814f63a4b823b2372ee11a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/36448fd7730649eb8c9a4d00fb105e8e] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=47.8 K 2024-11-27T00:16:14,129 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting eac398e47f0a48e7951991f3a505c960, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732666566054 2024-11-27T00:16:14,129 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting fb4928c0671845a38300cee85dca1f36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732666566761 2024-11-27T00:16:14,129 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ea4b76b14814f63a4b823b2372ee11a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732666568000 2024-11-27T00:16:14,129 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 36448fd7730649eb8c9a4d00fb105e8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666570206 2024-11-27T00:16:14,137 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:14,137 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/738d43424032415b9a0735494b1e2f53 is 50, key is test_row_0/C:col10/1732666570220/Put/seqid=0 2024-11-27T00:16:14,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742223_1399 (size=12629) 2024-11-27T00:16:14,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:14,155 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cc3cf87aa5e34e598e03e20fdce2778f_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cc3cf87aa5e34e598e03e20fdce2778f_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:14,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/16979afd19a242b585adea6f03615ffe, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:14,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/16979afd19a242b585adea6f03615ffe is 175, key is test_row_0/A:col10/1732666572470/Put/seqid=0 2024-11-27T00:16:14,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742224_1400 (size=31105) 2024-11-27T00:16:14,530 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/8d644a78f6664da28ea8f854f3db9994 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8d644a78f6664da28ea8f854f3db9994 2024-11-27T00:16:14,536 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/A of 45f2ab312c51b83f191580f6a2814191 into 8d644a78f6664da28ea8f854f3db9994(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:14,536 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:14,536 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/A, priority=12, startTime=1732666573677; duration=0sec 2024-11-27T00:16:14,536 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:14,536 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:A 2024-11-27T00:16:14,551 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/738d43424032415b9a0735494b1e2f53 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/738d43424032415b9a0735494b1e2f53 2024-11-27T00:16:14,558 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/C of 45f2ab312c51b83f191580f6a2814191 into 738d43424032415b9a0735494b1e2f53(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:14,558 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:14,558 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/C, priority=12, startTime=1732666573677; duration=0sec 2024-11-27T00:16:14,558 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:14,558 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:C 2024-11-27T00:16:14,561 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/16979afd19a242b585adea6f03615ffe 2024-11-27T00:16:14,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/ae3286e65434486bac1f97858a335239 is 50, key is test_row_0/B:col10/1732666572470/Put/seqid=0 2024-11-27T00:16:14,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742225_1401 (size=12151) 2024-11-27T00:16:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T00:16:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:14,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666634731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:14,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666634738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:14,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666634844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666634844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:14,974 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/ae3286e65434486bac1f97858a335239 2024-11-27T00:16:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/876d3fedee6442e7888c14840c027af0 is 50, key is test_row_0/C:col10/1732666572470/Put/seqid=0 2024-11-27T00:16:14,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742226_1402 (size=12151) 2024-11-27T00:16:14,987 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/876d3fedee6442e7888c14840c027af0 2024-11-27T00:16:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/16979afd19a242b585adea6f03615ffe as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/16979afd19a242b585adea6f03615ffe 2024-11-27T00:16:15,000 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/16979afd19a242b585adea6f03615ffe, entries=150, sequenceid=252, filesize=30.4 K 2024-11-27T00:16:15,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/ae3286e65434486bac1f97858a335239 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ae3286e65434486bac1f97858a335239 2024-11-27T00:16:15,008 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ae3286e65434486bac1f97858a335239, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T00:16:15,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/876d3fedee6442e7888c14840c027af0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/876d3fedee6442e7888c14840c027af0 2024-11-27T00:16:15,015 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/876d3fedee6442e7888c14840c027af0, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T00:16:15,016 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 45f2ab312c51b83f191580f6a2814191 in 1277ms, sequenceid=252, compaction requested=false 2024-11-27T00:16:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-27T00:16:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-27T00:16:15,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-27T00:16:15,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5010 sec 2024-11-27T00:16:15,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.5050 sec 2024-11-27T00:16:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:15,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:16:15,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:15,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:15,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:15,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:15,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:15,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:15,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112738916bf36c4c489187a501313a0da3b9_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666575059/Put/seqid=0 2024-11-27T00:16:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742227_1403 (size=17534) 2024-11-27T00:16:15,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666635112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:15,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666635119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:15,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666635234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666635263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:15,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666635444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:15,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666635473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:15,497 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:15,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112738916bf36c4c489187a501313a0da3b9_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738916bf36c4c489187a501313a0da3b9_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:15,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/1f18fec0733f4726a3be1a8c653900a5, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:15,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/1f18fec0733f4726a3be1a8c653900a5 is 175, key is test_row_0/A:col10/1732666575059/Put/seqid=0 2024-11-27T00:16:15,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742228_1404 (size=48639) 2024-11-27T00:16:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666635763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:15,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666635784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:15,911 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/1f18fec0733f4726a3be1a8c653900a5 2024-11-27T00:16:15,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/e2503e5c3ff74be0b140f0e8f3e7dbc8 is 50, key is test_row_0/B:col10/1732666575059/Put/seqid=0 2024-11-27T00:16:15,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742229_1405 (size=12301) 2024-11-27T00:16:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666636284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:16,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666636294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:16,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:16,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/e2503e5c3ff74be0b140f0e8f3e7dbc8 2024-11-27T00:16:16,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/9743424d07b34b2084071a139a707a56 is 50, key is test_row_0/C:col10/1732666575059/Put/seqid=0 2024-11-27T00:16:16,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742230_1406 (size=12301) 2024-11-27T00:16:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T00:16:16,684 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:16,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-27T00:16:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T00:16:16,686 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:16,687 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:16,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:16,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/9743424d07b34b2084071a139a707a56 2024-11-27T00:16:16,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/1f18fec0733f4726a3be1a8c653900a5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/1f18fec0733f4726a3be1a8c653900a5 2024-11-27T00:16:16,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/1f18fec0733f4726a3be1a8c653900a5, entries=250, sequenceid=272, filesize=47.5 K 2024-11-27T00:16:16,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/e2503e5c3ff74be0b140f0e8f3e7dbc8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e2503e5c3ff74be0b140f0e8f3e7dbc8 2024-11-27T00:16:16,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e2503e5c3ff74be0b140f0e8f3e7dbc8, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T00:16:16,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/9743424d07b34b2084071a139a707a56 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/9743424d07b34b2084071a139a707a56 2024-11-27T00:16:16,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/9743424d07b34b2084071a139a707a56, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T00:16:16,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 45f2ab312c51b83f191580f6a2814191 in 1706ms, sequenceid=272, compaction requested=true 2024-11-27T00:16:16,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:16,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:16,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:16,766 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:16,767 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:16,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:16,768 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:16,768 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/B is initiating minor compaction (all files) 2024-11-27T00:16:16,768 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/B in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:16,768 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/622c4ba607414260af28fae4d715f23a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ae3286e65434486bac1f97858a335239, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e2503e5c3ff74be0b140f0e8f3e7dbc8] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=36.2 K 2024-11-27T00:16:16,768 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111327 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:16,768 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/A is initiating minor compaction (all files) 2024-11-27T00:16:16,768 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/A in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:16,768 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8d644a78f6664da28ea8f854f3db9994, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/16979afd19a242b585adea6f03615ffe, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/1f18fec0733f4726a3be1a8c653900a5] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=108.7 K 2024-11-27T00:16:16,768 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:16,768 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8d644a78f6664da28ea8f854f3db9994, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/16979afd19a242b585adea6f03615ffe, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/1f18fec0733f4726a3be1a8c653900a5] 2024-11-27T00:16:16,769 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 622c4ba607414260af28fae4d715f23a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666570206 2024-11-27T00:16:16,769 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d644a78f6664da28ea8f854f3db9994, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666570206 2024-11-27T00:16:16,769 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3286e65434486bac1f97858a335239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732666572468 2024-11-27T00:16:16,769 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e2503e5c3ff74be0b140f0e8f3e7dbc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732666574726 2024-11-27T00:16:16,769 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16979afd19a242b585adea6f03615ffe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732666572468 2024-11-27T00:16:16,770 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f18fec0733f4726a3be1a8c653900a5, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732666574690 2024-11-27T00:16:16,776 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:16,777 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:16,778 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/b9fc4dd00dec4b52834d79b5efd819d0 is 50, key is test_row_0/B:col10/1732666575059/Put/seqid=0 2024-11-27T00:16:16,778 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411274d38b2f3e7cb4b5bbc318b113d7a30b7_45f2ab312c51b83f191580f6a2814191 store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:16,780 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411274d38b2f3e7cb4b5bbc318b113d7a30b7_45f2ab312c51b83f191580f6a2814191, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:16,781 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274d38b2f3e7cb4b5bbc318b113d7a30b7_45f2ab312c51b83f191580f6a2814191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:16,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742231_1407 (size=12881) 2024-11-27T00:16:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T00:16:16,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742232_1408 (size=4469) 2024-11-27T00:16:16,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T00:16:16,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:16,841 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:16:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273bf3836ccbb24a2da9758d999d25b7b9_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666575083/Put/seqid=0 2024-11-27T00:16:16,861 WARN [regionserver/f036bc4986ea:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-27T00:16:16,862 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/471e58b7f73b41fd8f07dbfa431df52d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/c350c1401d8a4ebea0b7febbe70ed050, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/618be1258fcb4a1798632dd12bb4daaf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/f0f8eecee4194c91a46e06e875cb2a6d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/857c53803635450297c20d2780f749fb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/96fb37ab02f149f2985352e45ab78f05, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8b763157f7e94bd7b497c77b9c4c1ab5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/df4e9482e7ab44e6be9df3899df2e4a6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8de0a0b7ac274770a620cbdaeca1b0f4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/21e3d944203e44b4985156d89c0ce9ba, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0a0760271b184537bbe92aa4e1903017, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/40bba7711b8c4553a28055c202f7de37, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/926d10436482481584ffb631f3ebe675, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/bb299df137f343cdba8413b68a0cd18c] to archive 2024-11-27T00:16:16,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:16,865 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/471e58b7f73b41fd8f07dbfa431df52d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/471e58b7f73b41fd8f07dbfa431df52d 2024-11-27T00:16:16,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742233_1409 (size=12454) 2024-11-27T00:16:16,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/c350c1401d8a4ebea0b7febbe70ed050 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/c350c1401d8a4ebea0b7febbe70ed050 2024-11-27T00:16:16,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:16,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/618be1258fcb4a1798632dd12bb4daaf to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/618be1258fcb4a1798632dd12bb4daaf 2024-11-27T00:16:16,884 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273bf3836ccbb24a2da9758d999d25b7b9_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273bf3836ccbb24a2da9758d999d25b7b9_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:16,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/0baa403622804040ad6f86fa54bfd1b4, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/0baa403622804040ad6f86fa54bfd1b4 is 175, key is test_row_0/A:col10/1732666575083/Put/seqid=0 2024-11-27T00:16:16,892 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/f0f8eecee4194c91a46e06e875cb2a6d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/f0f8eecee4194c91a46e06e875cb2a6d 2024-11-27T00:16:16,897 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/857c53803635450297c20d2780f749fb to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/857c53803635450297c20d2780f749fb 2024-11-27T00:16:16,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/96fb37ab02f149f2985352e45ab78f05 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/96fb37ab02f149f2985352e45ab78f05 2024-11-27T00:16:16,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8b763157f7e94bd7b497c77b9c4c1ab5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8b763157f7e94bd7b497c77b9c4c1ab5 2024-11-27T00:16:16,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/df4e9482e7ab44e6be9df3899df2e4a6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/df4e9482e7ab44e6be9df3899df2e4a6 2024-11-27T00:16:16,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8de0a0b7ac274770a620cbdaeca1b0f4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8de0a0b7ac274770a620cbdaeca1b0f4 2024-11-27T00:16:16,914 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/21e3d944203e44b4985156d89c0ce9ba to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/21e3d944203e44b4985156d89c0ce9ba 2024-11-27T00:16:16,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0a0760271b184537bbe92aa4e1903017 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0a0760271b184537bbe92aa4e1903017 2024-11-27T00:16:16,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/40bba7711b8c4553a28055c202f7de37 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/40bba7711b8c4553a28055c202f7de37 2024-11-27T00:16:16,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742234_1410 (size=31255) 2024-11-27T00:16:16,917 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/0baa403622804040ad6f86fa54bfd1b4 2024-11-27T00:16:16,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/926d10436482481584ffb631f3ebe675 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/926d10436482481584ffb631f3ebe675 2024-11-27T00:16:16,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/bb299df137f343cdba8413b68a0cd18c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/bb299df137f343cdba8413b68a0cd18c 2024-11-27T00:16:16,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4b1fb740887f416eabb8abc6c0e21305, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/93e50463ce624eeb8be7a1b09d6299eb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/50888595e8864e1aa238700693a10358, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/22a75195e326416a817b21dcea11a88b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/2b1892740136466097e97ef28fbb6339, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e9d599f4a2dc4484abcb4fd0bf5841ba, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0d6e744bc21649aa8485632281f69ef5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/d47c5f778b5b4a40887b0799fc29b90d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0c4eb457ca3141c9827074e302838aff, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/495fe36c9786485886800c01d82f9fa6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/24c1a29b25c54cf3bb8ccdf098876834, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4e941b92a7cb4ac9b8c382442677c146, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ab9c4f4a512c451d83c235f413960511, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/cc23ff43ecaa482aa5554eccb5227fbf] to archive 2024-11-27T00:16:16,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:16,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4b1fb740887f416eabb8abc6c0e21305 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4b1fb740887f416eabb8abc6c0e21305 2024-11-27T00:16:16,937 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/93e50463ce624eeb8be7a1b09d6299eb to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/93e50463ce624eeb8be7a1b09d6299eb 2024-11-27T00:16:16,938 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/50888595e8864e1aa238700693a10358 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/50888595e8864e1aa238700693a10358 2024-11-27T00:16:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/6b3c90eeb31e45768ca2424454d70ef6 is 50, key is test_row_0/B:col10/1732666575083/Put/seqid=0 2024-11-27T00:16:16,939 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/22a75195e326416a817b21dcea11a88b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/22a75195e326416a817b21dcea11a88b 2024-11-27T00:16:16,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/2b1892740136466097e97ef28fbb6339 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/2b1892740136466097e97ef28fbb6339 2024-11-27T00:16:16,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e9d599f4a2dc4484abcb4fd0bf5841ba to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e9d599f4a2dc4484abcb4fd0bf5841ba 2024-11-27T00:16:16,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0d6e744bc21649aa8485632281f69ef5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0d6e744bc21649aa8485632281f69ef5 2024-11-27T00:16:16,951 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/d47c5f778b5b4a40887b0799fc29b90d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/d47c5f778b5b4a40887b0799fc29b90d 2024-11-27T00:16:16,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0c4eb457ca3141c9827074e302838aff to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/0c4eb457ca3141c9827074e302838aff 2024-11-27T00:16:16,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/495fe36c9786485886800c01d82f9fa6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/495fe36c9786485886800c01d82f9fa6 2024-11-27T00:16:16,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742235_1411 (size=12301) 2024-11-27T00:16:16,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/24c1a29b25c54cf3bb8ccdf098876834 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/24c1a29b25c54cf3bb8ccdf098876834 2024-11-27T00:16:16,960 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/6b3c90eeb31e45768ca2424454d70ef6 2024-11-27T00:16:16,961 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4e941b92a7cb4ac9b8c382442677c146 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/4e941b92a7cb4ac9b8c382442677c146 2024-11-27T00:16:16,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ab9c4f4a512c451d83c235f413960511 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ab9c4f4a512c451d83c235f413960511 2024-11-27T00:16:16,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/cc23ff43ecaa482aa5554eccb5227fbf to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/cc23ff43ecaa482aa5554eccb5227fbf 2024-11-27T00:16:16,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3e54e1969d294476a9d6c35761406e56, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/29aea5eafb3340f78473bd30642c3b67, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/06d2071d5f91495dbf991a4aa2f5313a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b29f7bcab369406ba84fdd4d63f2c849, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/e7b5148b9c2e4f5aa35b82964b9fc290, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/bc303de9317a43149a22b75694e6beb4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a16b3960f72a4e6bbf06e3c05655cf92, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/51a503a962714fd18cebcc9ed479f6df, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/82f96bc17838497ebc036f8a3a472720, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/eac398e47f0a48e7951991f3a505c960, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b60f05ebdd4741da975b1c3011c1d1a5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/fb4928c0671845a38300cee85dca1f36, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3ea4b76b14814f63a4b823b2372ee11a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/36448fd7730649eb8c9a4d00fb105e8e] to archive 2024-11-27T00:16:16,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:16,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3e54e1969d294476a9d6c35761406e56 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3e54e1969d294476a9d6c35761406e56 2024-11-27T00:16:16,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/f5d96d499bf049b9bb44a8641a7ec262 is 50, key is test_row_0/C:col10/1732666575083/Put/seqid=0 2024-11-27T00:16:16,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/29aea5eafb3340f78473bd30642c3b67 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/29aea5eafb3340f78473bd30642c3b67 2024-11-27T00:16:16,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/06d2071d5f91495dbf991a4aa2f5313a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/06d2071d5f91495dbf991a4aa2f5313a 2024-11-27T00:16:16,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b29f7bcab369406ba84fdd4d63f2c849 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b29f7bcab369406ba84fdd4d63f2c849 2024-11-27T00:16:16,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742236_1412 (size=12301) 2024-11-27T00:16:16,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/e7b5148b9c2e4f5aa35b82964b9fc290 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/e7b5148b9c2e4f5aa35b82964b9fc290 2024-11-27T00:16:16,978 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/f5d96d499bf049b9bb44a8641a7ec262 2024-11-27T00:16:16,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/bc303de9317a43149a22b75694e6beb4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/bc303de9317a43149a22b75694e6beb4 2024-11-27T00:16:16,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a16b3960f72a4e6bbf06e3c05655cf92 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a16b3960f72a4e6bbf06e3c05655cf92 2024-11-27T00:16:16,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/51a503a962714fd18cebcc9ed479f6df to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/51a503a962714fd18cebcc9ed479f6df 2024-11-27T00:16:16,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/82f96bc17838497ebc036f8a3a472720 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/82f96bc17838497ebc036f8a3a472720 2024-11-27T00:16:16,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/eac398e47f0a48e7951991f3a505c960 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/eac398e47f0a48e7951991f3a505c960 2024-11-27T00:16:16,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b60f05ebdd4741da975b1c3011c1d1a5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/b60f05ebdd4741da975b1c3011c1d1a5 2024-11-27T00:16:16,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/0baa403622804040ad6f86fa54bfd1b4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0baa403622804040ad6f86fa54bfd1b4 2024-11-27T00:16:16,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/fb4928c0671845a38300cee85dca1f36 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/fb4928c0671845a38300cee85dca1f36 2024-11-27T00:16:16,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3ea4b76b14814f63a4b823b2372ee11a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/3ea4b76b14814f63a4b823b2372ee11a 2024-11-27T00:16:16,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f036bc4986ea:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/36448fd7730649eb8c9a4d00fb105e8e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/36448fd7730649eb8c9a4d00fb105e8e 2024-11-27T00:16:16,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0baa403622804040ad6f86fa54bfd1b4, entries=150, sequenceid=287, filesize=30.5 K 2024-11-27T00:16:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:16,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/6b3c90eeb31e45768ca2424454d70ef6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/6b3c90eeb31e45768ca2424454d70ef6 2024-11-27T00:16:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,001 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/6b3c90eeb31e45768ca2424454d70ef6, entries=150, sequenceid=287, filesize=12.0 K 2024-11-27T00:16:17,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/f5d96d499bf049b9bb44a8641a7ec262 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/f5d96d499bf049b9bb44a8641a7ec262 2024-11-27T00:16:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T00:16:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,005 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/f5d96d499bf049b9bb44a8641a7ec262, entries=150, sequenceid=287, filesize=12.0 K 2024-11-27T00:16:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,006 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 45f2ab312c51b83f191580f6a2814191 in 165ms, sequenceid=287, compaction requested=true 2024-11-27T00:16:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:17,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-27T00:16:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-27T00:16:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-27T00:16:17,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 321 msec 2024-11-27T00:16:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 327 msec 2024-11-27T00:16:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/b9fc4dd00dec4b52834d79b5efd819d0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/b9fc4dd00dec4b52834d79b5efd819d0 2024-11-27T00:16:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,199 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,199 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/030059a2228642c08a72c5c5fa3dbcc3 is 175, key is test_row_0/A:col10/1732666575059/Put/seqid=0 2024-11-27T00:16:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742237_1413 (size=31358) 2024-11-27T00:16:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,230 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/B of 45f2ab312c51b83f191580f6a2814191 into b9fc4dd00dec4b52834d79b5efd819d0(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:17,230 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:17,230 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/B, priority=13, startTime=1732666576766; duration=0sec 2024-11-27T00:16:17,230 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,230 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:B 2024-11-27T00:16:17,230 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,232 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,232 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/C is initiating minor compaction (all files) 2024-11-27T00:16:17,232 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/C in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,232 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/738d43424032415b9a0735494b1e2f53, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/876d3fedee6442e7888c14840c027af0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/9743424d07b34b2084071a139a707a56, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/f5d96d499bf049b9bb44a8641a7ec262] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=48.2 K 2024-11-27T00:16:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,233 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 738d43424032415b9a0735494b1e2f53, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666570206 2024-11-27T00:16:17,233 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 876d3fedee6442e7888c14840c027af0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732666572468 2024-11-27T00:16:17,233 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9743424d07b34b2084071a139a707a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732666574726 2024-11-27T00:16:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,234 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f5d96d499bf049b9bb44a8641a7ec262, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732666575083 2024-11-27T00:16:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,246 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:17,247 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/07bcdb51c9014f89aa42e8ea19f36050 is 50, key is test_row_0/C:col10/1732666575083/Put/seqid=0 2024-11-27T00:16:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742238_1414 (size=12439) 2024-11-27T00:16:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,292 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/07bcdb51c9014f89aa42e8ea19f36050 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/07bcdb51c9014f89aa42e8ea19f36050 2024-11-27T00:16:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,297 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/C of 45f2ab312c51b83f191580f6a2814191 into 07bcdb51c9014f89aa42e8ea19f36050(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:17,297 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:17,297 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/C, priority=12, startTime=1732666576767; duration=0sec 2024-11-27T00:16:17,297 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:17,297 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:C 2024-11-27T00:16:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T00:16:17,315 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,315 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-27T00:16:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T00:16:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,317 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:17,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T00:16:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:17,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:16:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:17,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:17,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279b9e10df9dad403e9e94742d93717ae5_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666577428/Put/seqid=0 2024-11-27T00:16:17,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742239_1415 (size=14994) 2024-11-27T00:16:17,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:17,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:17,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732666577494, max timestamp=1732666577531Num uniques 859; 2024-11-27T00:16:17,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732666577494, max timestamp=1732666577531Num uniques 859; 2024-11-27T00:16:17,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:17,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732666577494, max timestamp=1732666577531Num uniques 859; 2024-11-27T00:16:17,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:17,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:17,608 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/030059a2228642c08a72c5c5fa3dbcc3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/030059a2228642c08a72c5c5fa3dbcc3 2024-11-27T00:16:17,615 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/A of 45f2ab312c51b83f191580f6a2814191 into 030059a2228642c08a72c5c5fa3dbcc3(size=30.6 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:17,615 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:17,615 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/A, priority=13, startTime=1732666576766; duration=0sec 2024-11-27T00:16:17,615 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:17,615 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:A 2024-11-27T00:16:17,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666637607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:17,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:17,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666637609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:17,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:17,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:17,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T00:16:17,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666637724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:17,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666637734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:17,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:17,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:17,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,849 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-27T00:16:17,849 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:17,849 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:17,850 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,853 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-27T00:16:17,853 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:17,853 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,853 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:17,858 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-27T00:16:17,859 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:17,859 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,859 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:17,861 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-27T00:16:17,861 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:17,861 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,861 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:17,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:17,866 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279b9e10df9dad403e9e94742d93717ae5_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279b9e10df9dad403e9e94742d93717ae5_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:17,867 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/54eb781177ef4548a9d413874fdbcd09, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:17,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/54eb781177ef4548a9d413874fdbcd09 is 175, key is test_row_0/A:col10/1732666577428/Put/seqid=0 2024-11-27T00:16:17,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742240_1416 (size=39949) 2024-11-27T00:16:17,871 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-27T00:16:17,871 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:17,871 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,871 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:17,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:17,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666637934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:17,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:17,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T00:16:17,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666637954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:18,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666638244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:18,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666638265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:18,272 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/54eb781177ef4548a9d413874fdbcd09 2024-11-27T00:16:18,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/332594c8db674665b2ea08dfc27647f0 is 50, key is test_row_0/B:col10/1732666577428/Put/seqid=0 2024-11-27T00:16:18,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742241_1417 (size=12301) 2024-11-27T00:16:18,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T00:16:18,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/332594c8db674665b2ea08dfc27647f0 2024-11-27T00:16:18,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/662476bc43d744e69a9e32e880da0d30 is 50, key is test_row_0/C:col10/1732666577428/Put/seqid=0 2024-11-27T00:16:18,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742242_1418 (size=12301) 2024-11-27T00:16:18,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:18,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59634 deadline: 1732666638724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59656 deadline: 1732666638724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:18,724 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:18,724 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18290 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:18,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59612 deadline: 1732666638754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:18,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59636 deadline: 1732666638774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:18,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59584 deadline: 1732666638805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:18,806 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 45f2ab312c51b83f191580f6a2814191 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18363 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:16:18,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:18,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:19,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:19,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:19,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/662476bc43d744e69a9e32e880da0d30 2024-11-27T00:16:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/54eb781177ef4548a9d413874fdbcd09 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/54eb781177ef4548a9d413874fdbcd09 2024-11-27T00:16:19,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/54eb781177ef4548a9d413874fdbcd09, entries=200, sequenceid=299, filesize=39.0 K 2024-11-27T00:16:19,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/332594c8db674665b2ea08dfc27647f0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/332594c8db674665b2ea08dfc27647f0 2024-11-27T00:16:19,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/332594c8db674665b2ea08dfc27647f0, entries=150, sequenceid=299, filesize=12.0 K 2024-11-27T00:16:19,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/662476bc43d744e69a9e32e880da0d30 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/662476bc43d744e69a9e32e880da0d30 2024-11-27T00:16:19,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/662476bc43d744e69a9e32e880da0d30, entries=150, sequenceid=299, filesize=12.0 K 2024-11-27T00:16:19,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 45f2ab312c51b83f191580f6a2814191 in 1667ms, sequenceid=299, compaction requested=true 2024-11-27T00:16:19,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:19,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:19,114 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:19,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:19,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:19,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:19,114 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:19,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45f2ab312c51b83f191580f6a2814191:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:19,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/B is initiating minor compaction (all files) 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 45f2ab312c51b83f191580f6a2814191/A is initiating minor compaction (all files) 2024-11-27T00:16:19,115 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/A in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:19,115 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45f2ab312c51b83f191580f6a2814191/B in TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:19,115 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/030059a2228642c08a72c5c5fa3dbcc3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0baa403622804040ad6f86fa54bfd1b4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/54eb781177ef4548a9d413874fdbcd09] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=100.2 K 2024-11-27T00:16:19,115 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:19,115 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/b9fc4dd00dec4b52834d79b5efd819d0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/6b3c90eeb31e45768ca2424454d70ef6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/332594c8db674665b2ea08dfc27647f0] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp, totalSize=36.6 K 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/030059a2228642c08a72c5c5fa3dbcc3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0baa403622804040ad6f86fa54bfd1b4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/54eb781177ef4548a9d413874fdbcd09] 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 030059a2228642c08a72c5c5fa3dbcc3, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732666574726 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b9fc4dd00dec4b52834d79b5efd819d0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732666574726 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0baa403622804040ad6f86fa54bfd1b4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732666575083 2024-11-27T00:16:19,115 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3c90eeb31e45768ca2424454d70ef6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732666575083 2024-11-27T00:16:19,116 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 332594c8db674665b2ea08dfc27647f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732666577406 2024-11-27T00:16:19,116 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54eb781177ef4548a9d413874fdbcd09, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732666577406 2024-11-27T00:16:19,121 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:19,122 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:19,122 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/f7ae4e65119d4e2ebcac0f3658ec75fd is 50, key is test_row_0/B:col10/1732666577428/Put/seqid=0 2024-11-27T00:16:19,123 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127858c9147892740e686efd742b6679028_45f2ab312c51b83f191580f6a2814191 store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:19,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742243_1419 (size=12507) 2024-11-27T00:16:19,128 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127858c9147892740e686efd742b6679028_45f2ab312c51b83f191580f6a2814191, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:19,128 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127858c9147892740e686efd742b6679028_45f2ab312c51b83f191580f6a2814191 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:19,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742244_1420 (size=4469) 2024-11-27T00:16:19,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T00:16:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:19,167 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T00:16:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:19,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f5daa9f413d246ddb1d60a53794fabe3_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_0/A:col10/1732666577608/Put/seqid=0 2024-11-27T00:16:19,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742245_1421 (size=12454) 2024-11-27T00:16:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T00:16:19,533 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45f2ab312c51b83f191580f6a2814191#A#compaction#355 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:19,534 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/cccd3767fd3d4b8caec1bfd527b96fc5 is 175, key is test_row_0/A:col10/1732666577428/Put/seqid=0 2024-11-27T00:16:19,536 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/f7ae4e65119d4e2ebcac0f3658ec75fd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/f7ae4e65119d4e2ebcac0f3658ec75fd 2024-11-27T00:16:19,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742246_1422 (size=31461) 2024-11-27T00:16:19,542 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/B of 45f2ab312c51b83f191580f6a2814191 into f7ae4e65119d4e2ebcac0f3658ec75fd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:19,542 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:19,542 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/B, priority=13, startTime=1732666579114; duration=0sec 2024-11-27T00:16:19,543 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:19,543 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:B 2024-11-27T00:16:19,543 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T00:16:19,543 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T00:16:19,543 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T00:16:19,544 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. because compaction request was cancelled 2024-11-27T00:16:19,544 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:C 2024-11-27T00:16:19,544 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/cccd3767fd3d4b8caec1bfd527b96fc5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/cccd3767fd3d4b8caec1bfd527b96fc5 2024-11-27T00:16:19,548 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45f2ab312c51b83f191580f6a2814191/A of 45f2ab312c51b83f191580f6a2814191 into cccd3767fd3d4b8caec1bfd527b96fc5(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:19,548 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:19,548 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., storeName=45f2ab312c51b83f191580f6a2814191/A, priority=13, startTime=1732666579114; duration=0sec 2024-11-27T00:16:19,549 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:19,549 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45f2ab312c51b83f191580f6a2814191:A 2024-11-27T00:16:19,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:19,582 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f5daa9f413d246ddb1d60a53794fabe3_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f5daa9f413d246ddb1d60a53794fabe3_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:19,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/25839e817a884715961a3370e6b07993, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:19,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/25839e817a884715961a3370e6b07993 is 175, key is test_row_0/A:col10/1732666577608/Put/seqid=0 2024-11-27T00:16:19,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742247_1423 (size=31255) 2024-11-27T00:16:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:19,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. as already flushing 2024-11-27T00:16:19,777 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-27T00:16:19,777 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:19,777 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:19,778 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:19,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:19,787 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-27T00:16:19,788 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:19,788 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:19,788 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:19,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:19,990 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/25839e817a884715961a3370e6b07993 2024-11-27T00:16:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/80fbec84f15e49ec94e1d0f571638af0 is 50, key is test_row_0/B:col10/1732666577608/Put/seqid=0 2024-11-27T00:16:20,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742248_1424 (size=12301) 2024-11-27T00:16:20,409 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/80fbec84f15e49ec94e1d0f571638af0 2024-11-27T00:16:20,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/869bb567bf14403a836b296484a233b7 is 50, key is test_row_0/C:col10/1732666577608/Put/seqid=0 2024-11-27T00:16:20,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742249_1425 (size=12301) 2024-11-27T00:16:20,829 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/869bb567bf14403a836b296484a233b7 2024-11-27T00:16:20,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/25839e817a884715961a3370e6b07993 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/25839e817a884715961a3370e6b07993 2024-11-27T00:16:20,844 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/25839e817a884715961a3370e6b07993, entries=150, sequenceid=336, filesize=30.5 K 2024-11-27T00:16:20,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/80fbec84f15e49ec94e1d0f571638af0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/80fbec84f15e49ec94e1d0f571638af0 2024-11-27T00:16:20,849 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/80fbec84f15e49ec94e1d0f571638af0, entries=150, sequenceid=336, filesize=12.0 K 2024-11-27T00:16:20,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/869bb567bf14403a836b296484a233b7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/869bb567bf14403a836b296484a233b7 2024-11-27T00:16:20,853 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/869bb567bf14403a836b296484a233b7, entries=150, sequenceid=336, filesize=12.0 K 2024-11-27T00:16:20,854 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=13.42 KB/13740 for 45f2ab312c51b83f191580f6a2814191 in 1687ms, sequenceid=336, compaction requested=true 2024-11-27T00:16:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 45f2ab312c51b83f191580f6a2814191: 2024-11-27T00:16:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-27T00:16:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-27T00:16:20,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-27T00:16:20,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5370 sec 2024-11-27T00:16:20,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.5420 sec 2024-11-27T00:16:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T00:16:21,485 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:28,739 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-27T00:16:28,739 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:28,740 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:28,740 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:28,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:28,747 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-27T00:16:28,747 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:28,747 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:28,747 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:28,747 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:28,837 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-27T00:16:28,837 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:28,837 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:28,837 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:28,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1042 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3125 rows 2024-11-27T00:16:28,838 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1036 2024-11-27T00:16:28,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3105 rows 2024-11-27T00:16:28,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1041 2024-11-27T00:16:28,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3114 rows 2024-11-27T00:16:28,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1035 2024-11-27T00:16:28,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3101 rows 2024-11-27T00:16:28,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1041 2024-11-27T00:16:28,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3121 rows 2024-11-27T00:16:28,839 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T00:16:28,839 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:16:28,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:28,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:28,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:28,841 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T00:16:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T00:16:28,848 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666588847"}]},"ts":"1732666588847"} 2024-11-27T00:16:28,849 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T00:16:28,849 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T00:16:28,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:16:28,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, UNASSIGN}] 2024-11-27T00:16:28,852 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, UNASSIGN 2024-11-27T00:16:28,853 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=45f2ab312c51b83f191580f6a2814191, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:28,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, UNASSIGN because future has completed 2024-11-27T00:16:28,854 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:16:28,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:16:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T00:16:29,007 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:29,007 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:16:29,007 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 45f2ab312c51b83f191580f6a2814191, disabling compactions & flushes 2024-11-27T00:16:29,007 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:29,007 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:29,007 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. after waiting 0 ms 2024-11-27T00:16:29,007 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:29,007 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 45f2ab312c51b83f191580f6a2814191 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-27T00:16:29,008 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=A 2024-11-27T00:16:29,008 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:29,008 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=B 2024-11-27T00:16:29,008 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:29,008 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45f2ab312c51b83f191580f6a2814191, store=C 2024-11-27T00:16:29,008 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:29,018 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0eb2559cfc846cb9c4ba658dc27b5c4_45f2ab312c51b83f191580f6a2814191 is 50, key is test_row_1/A:col10/1732666588736/Put/seqid=0 2024-11-27T00:16:29,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742250_1426 (size=9914) 2024-11-27T00:16:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T00:16:29,422 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:29,426 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0eb2559cfc846cb9c4ba658dc27b5c4_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0eb2559cfc846cb9c4ba658dc27b5c4_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:29,426 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/af1a4248472f4a55a6bf7fc001a0420b, store: [table=TestAcidGuarantees family=A region=45f2ab312c51b83f191580f6a2814191] 2024-11-27T00:16:29,427 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/af1a4248472f4a55a6bf7fc001a0420b is 175, key is test_row_1/A:col10/1732666588736/Put/seqid=0 2024-11-27T00:16:29,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742251_1427 (size=22561) 2024-11-27T00:16:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T00:16:29,830 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/af1a4248472f4a55a6bf7fc001a0420b 2024-11-27T00:16:29,837 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/1549242d97624ef8bf0e8ecd2e9d2494 is 50, key is test_row_1/B:col10/1732666588736/Put/seqid=0 2024-11-27T00:16:29,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742252_1428 (size=9857) 2024-11-27T00:16:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T00:16:30,243 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/1549242d97624ef8bf0e8ecd2e9d2494 2024-11-27T00:16:30,255 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/a7f664ca43d14cbba060e2bd609f2aac is 50, key is test_row_1/C:col10/1732666588736/Put/seqid=0 2024-11-27T00:16:30,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742253_1429 (size=9857) 2024-11-27T00:16:30,661 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/a7f664ca43d14cbba060e2bd609f2aac 2024-11-27T00:16:30,671 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/A/af1a4248472f4a55a6bf7fc001a0420b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/af1a4248472f4a55a6bf7fc001a0420b 2024-11-27T00:16:30,674 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/af1a4248472f4a55a6bf7fc001a0420b, entries=100, sequenceid=346, filesize=22.0 K 2024-11-27T00:16:30,675 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/B/1549242d97624ef8bf0e8ecd2e9d2494 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/1549242d97624ef8bf0e8ecd2e9d2494 2024-11-27T00:16:30,679 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/1549242d97624ef8bf0e8ecd2e9d2494, entries=100, sequenceid=346, filesize=9.6 K 2024-11-27T00:16:30,679 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/.tmp/C/a7f664ca43d14cbba060e2bd609f2aac as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a7f664ca43d14cbba060e2bd609f2aac 2024-11-27T00:16:30,683 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a7f664ca43d14cbba060e2bd609f2aac, entries=100, sequenceid=346, filesize=9.6 K 2024-11-27T00:16:30,684 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 45f2ab312c51b83f191580f6a2814191 in 1677ms, sequenceid=346, compaction requested=true 2024-11-27T00:16:30,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8d644a78f6664da28ea8f854f3db9994, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/16979afd19a242b585adea6f03615ffe, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/1f18fec0733f4726a3be1a8c653900a5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/030059a2228642c08a72c5c5fa3dbcc3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0baa403622804040ad6f86fa54bfd1b4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/54eb781177ef4548a9d413874fdbcd09] to archive 2024-11-27T00:16:30,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:30,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8d644a78f6664da28ea8f854f3db9994 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/8d644a78f6664da28ea8f854f3db9994 2024-11-27T00:16:30,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/16979afd19a242b585adea6f03615ffe to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/16979afd19a242b585adea6f03615ffe 2024-11-27T00:16:30,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/1f18fec0733f4726a3be1a8c653900a5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/1f18fec0733f4726a3be1a8c653900a5 2024-11-27T00:16:30,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/030059a2228642c08a72c5c5fa3dbcc3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/030059a2228642c08a72c5c5fa3dbcc3 2024-11-27T00:16:30,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0baa403622804040ad6f86fa54bfd1b4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/0baa403622804040ad6f86fa54bfd1b4 2024-11-27T00:16:30,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/54eb781177ef4548a9d413874fdbcd09 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/54eb781177ef4548a9d413874fdbcd09 2024-11-27T00:16:30,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/622c4ba607414260af28fae4d715f23a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ae3286e65434486bac1f97858a335239, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/b9fc4dd00dec4b52834d79b5efd819d0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e2503e5c3ff74be0b140f0e8f3e7dbc8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/6b3c90eeb31e45768ca2424454d70ef6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/332594c8db674665b2ea08dfc27647f0] to archive 2024-11-27T00:16:30,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:30,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/622c4ba607414260af28fae4d715f23a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/622c4ba607414260af28fae4d715f23a 2024-11-27T00:16:30,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ae3286e65434486bac1f97858a335239 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/ae3286e65434486bac1f97858a335239 2024-11-27T00:16:30,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/b9fc4dd00dec4b52834d79b5efd819d0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/b9fc4dd00dec4b52834d79b5efd819d0 2024-11-27T00:16:30,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e2503e5c3ff74be0b140f0e8f3e7dbc8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/e2503e5c3ff74be0b140f0e8f3e7dbc8 2024-11-27T00:16:30,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/6b3c90eeb31e45768ca2424454d70ef6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/6b3c90eeb31e45768ca2424454d70ef6 2024-11-27T00:16:30,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/332594c8db674665b2ea08dfc27647f0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/332594c8db674665b2ea08dfc27647f0 2024-11-27T00:16:30,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/738d43424032415b9a0735494b1e2f53, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/876d3fedee6442e7888c14840c027af0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/9743424d07b34b2084071a139a707a56, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/f5d96d499bf049b9bb44a8641a7ec262] to archive 2024-11-27T00:16:30,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:30,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/738d43424032415b9a0735494b1e2f53 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/738d43424032415b9a0735494b1e2f53 2024-11-27T00:16:30,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/876d3fedee6442e7888c14840c027af0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/876d3fedee6442e7888c14840c027af0 2024-11-27T00:16:30,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/9743424d07b34b2084071a139a707a56 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/9743424d07b34b2084071a139a707a56 2024-11-27T00:16:30,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/f5d96d499bf049b9bb44a8641a7ec262 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/f5d96d499bf049b9bb44a8641a7ec262 2024-11-27T00:16:30,714 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=4 2024-11-27T00:16:30,714 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. 2024-11-27T00:16:30,714 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 45f2ab312c51b83f191580f6a2814191: Waiting for close lock at 1732666589007Running coprocessor pre-close hooks at 1732666589007Disabling compacts and flushes for region at 1732666589007Disabling writes for close at 1732666589007Obtaining lock to block concurrent updates at 1732666589007Preparing flush snapshotting stores in 45f2ab312c51b83f191580f6a2814191 at 1732666589007Finished memstore snapshotting TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732666589008 (+1 ms)Flushing stores of TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191. at 1732666589008Flushing 45f2ab312c51b83f191580f6a2814191/A: creating writer at 1732666589008Flushing mob file 45f2ab312c51b83f191580f6a2814191/A: appending metadata at 1732666589018 (+10 ms)Flushing mob file 45f2ab312c51b83f191580f6a2814191/A: closing flushed file at 1732666589018Flushing 45f2ab312c51b83f191580f6a2814191/A: appending metadata at 1732666589426 (+408 ms)Flushing 45f2ab312c51b83f191580f6a2814191/A: closing flushed file at 1732666589426Flushing 45f2ab312c51b83f191580f6a2814191/B: creating writer at 1732666589834 (+408 ms)Flushing 45f2ab312c51b83f191580f6a2814191/B: appending metadata at 1732666589836 (+2 ms)Flushing 45f2ab312c51b83f191580f6a2814191/B: closing flushed file at 1732666589836Flushing 45f2ab312c51b83f191580f6a2814191/C: creating writer at 1732666590251 (+415 ms)Flushing 45f2ab312c51b83f191580f6a2814191/C: appending metadata at 1732666590255 (+4 ms)Flushing 45f2ab312c51b83f191580f6a2814191/C: closing flushed file at 1732666590255Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4591b15e: reopening flushed file at 1732666590670 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fdbc0b6: reopening flushed file at 1732666590675 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@385e2979: reopening flushed file at 1732666590679 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 45f2ab312c51b83f191580f6a2814191 in 1677ms, sequenceid=346, compaction requested=true at 1732666590684 (+5 ms)Writing region close event to WAL at 1732666590712 (+28 ms)Running coprocessor post-close hooks at 1732666590714 (+2 ms)Closed at 1732666590714 2024-11-27T00:16:30,716 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:30,717 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=45f2ab312c51b83f191580f6a2814191, regionState=CLOSED 2024-11-27T00:16:30,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:16:30,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-27T00:16:30,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 45f2ab312c51b83f191580f6a2814191, server=f036bc4986ea,33881,1732666453890 in 1.8650 sec 2024-11-27T00:16:30,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-27T00:16:30,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45f2ab312c51b83f191580f6a2814191, UNASSIGN in 1.8680 sec 2024-11-27T00:16:30,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-27T00:16:30,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8730 sec 2024-11-27T00:16:30,726 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666590726"}]},"ts":"1732666590726"} 2024-11-27T00:16:30,727 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T00:16:30,727 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T00:16:30,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8870 sec 2024-11-27T00:16:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T00:16:30,994 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:30,994 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T00:16:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:30,996 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T00:16:30,996 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:30,999 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,001 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/recovered.edits] 2024-11-27T00:16:31,004 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/25839e817a884715961a3370e6b07993 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/25839e817a884715961a3370e6b07993 2024-11-27T00:16:31,006 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/af1a4248472f4a55a6bf7fc001a0420b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/af1a4248472f4a55a6bf7fc001a0420b 2024-11-27T00:16:31,007 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/cccd3767fd3d4b8caec1bfd527b96fc5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/A/cccd3767fd3d4b8caec1bfd527b96fc5 2024-11-27T00:16:31,009 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/1549242d97624ef8bf0e8ecd2e9d2494 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/1549242d97624ef8bf0e8ecd2e9d2494 2024-11-27T00:16:31,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/80fbec84f15e49ec94e1d0f571638af0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/80fbec84f15e49ec94e1d0f571638af0 2024-11-27T00:16:31,011 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/f7ae4e65119d4e2ebcac0f3658ec75fd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/B/f7ae4e65119d4e2ebcac0f3658ec75fd 2024-11-27T00:16:31,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/07bcdb51c9014f89aa42e8ea19f36050 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/07bcdb51c9014f89aa42e8ea19f36050 2024-11-27T00:16:31,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/662476bc43d744e69a9e32e880da0d30 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/662476bc43d744e69a9e32e880da0d30 2024-11-27T00:16:31,014 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/869bb567bf14403a836b296484a233b7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/869bb567bf14403a836b296484a233b7 2024-11-27T00:16:31,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a7f664ca43d14cbba060e2bd609f2aac to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/C/a7f664ca43d14cbba060e2bd609f2aac 2024-11-27T00:16:31,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/recovered.edits/349.seqid to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191/recovered.edits/349.seqid 2024-11-27T00:16:31,018 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,018 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T00:16:31,018 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T00:16:31,019 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T00:16:31,022 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112703b48658b6d447a094d76a0159c4c85c_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112703b48658b6d447a094d76a0159c4c85c_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,023 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271d7331a8a3554ce880b18c859be42b7f_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271d7331a8a3554ce880b18c859be42b7f_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,024 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738916bf36c4c489187a501313a0da3b9_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112738916bf36c4c489187a501313a0da3b9_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,024 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273bf3836ccbb24a2da9758d999d25b7b9_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273bf3836ccbb24a2da9758d999d25b7b9_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,025 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273fa65fa595b84dd381a3686d5e94efa1_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273fa65fa595b84dd381a3686d5e94efa1_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,026 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274cf99cbf350d4f4a823bd6da3f14a76f_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274cf99cbf350d4f4a823bd6da3f14a76f_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,026 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276ac5d5d17454404e95419440c466b679_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276ac5d5d17454404e95419440c466b679_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,027 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277bed36343d684265afa94b2569bc0bf3_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277bed36343d684265afa94b2569bc0bf3_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,028 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112788221186d4c64ee5a0b6d6b7ca0e7d3a_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112788221186d4c64ee5a0b6d6b7ca0e7d3a_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,029 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279b9e10df9dad403e9e94742d93717ae5_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279b9e10df9dad403e9e94742d93717ae5_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,030 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bcdc487937e642edae85294df3e15432_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bcdc487937e642edae85294df3e15432_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,030 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c384da6828914c5a9bab775b8f07a6b4_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c384da6828914c5a9bab775b8f07a6b4_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,031 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cc3cf87aa5e34e598e03e20fdce2778f_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cc3cf87aa5e34e598e03e20fdce2778f_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,032 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0eb2559cfc846cb9c4ba658dc27b5c4_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0eb2559cfc846cb9c4ba658dc27b5c4_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,033 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e227d0455781482f83e3f5b17fbe9e90_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e227d0455781482f83e3f5b17fbe9e90_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,034 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f5daa9f413d246ddb1d60a53794fabe3_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f5daa9f413d246ddb1d60a53794fabe3_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,035 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa8a35aaf150489aa15a6bb333a5e8b4_45f2ab312c51b83f191580f6a2814191 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa8a35aaf150489aa15a6bb333a5e8b4_45f2ab312c51b83f191580f6a2814191 2024-11-27T00:16:31,035 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T00:16:31,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:31,038 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T00:16:31,040 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T00:16:31,041 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:31,041 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T00:16:31,042 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732666591041"}]},"ts":"9223372036854775807"} 2024-11-27T00:16:31,043 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T00:16:31,043 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 45f2ab312c51b83f191580f6a2814191, NAME => 'TestAcidGuarantees,,1732666554706.45f2ab312c51b83f191580f6a2814191.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T00:16:31,043 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T00:16:31,044 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732666591043"}]},"ts":"9223372036854775807"} 2024-11-27T00:16:31,046 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T00:16:31,046 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:31,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-27T00:16:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T00:16:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T00:16:31,108 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:31,116 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 247) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=2229 (was 2940) 2024-11-27T00:16:31,125 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=2229 2024-11-27T00:16:31,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:16:31,126 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:16:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:31,128 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T00:16:31,128 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:31,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-27T00:16:31,129 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T00:16:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T00:16:31,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742254_1430 (size=1061) 2024-11-27T00:16:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T00:16:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T00:16:31,538 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 2024-11-27T00:16:31,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742255_1431 (size=53) 2024-11-27T00:16:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T00:16:31,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:16:31,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 969e097efa67bfc5f2c39f5533a48c02, disabling compactions & flushes 2024-11-27T00:16:31,971 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:31,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:31,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. after waiting 0 ms 2024-11-27T00:16:31,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:31,971 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:31,972 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 969e097efa67bfc5f2c39f5533a48c02: Waiting for close lock at 1732666591971Disabling compacts and flushes for region at 1732666591971Disabling writes for close at 1732666591971Writing region close event to WAL at 1732666591971Closed at 1732666591971 2024-11-27T00:16:31,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T00:16:31,975 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732666591974"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732666591974"}]},"ts":"1732666591974"} 2024-11-27T00:16:31,977 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T00:16:31,978 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T00:16:31,979 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666591978"}]},"ts":"1732666591978"} 2024-11-27T00:16:31,980 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T00:16:31,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, ASSIGN}] 2024-11-27T00:16:31,982 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, ASSIGN 2024-11-27T00:16:31,983 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, ASSIGN; state=OFFLINE, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=false 2024-11-27T00:16:32,134 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=969e097efa67bfc5f2c39f5533a48c02, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:32,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, ASSIGN because future has completed 2024-11-27T00:16:32,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:16:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T00:16:32,293 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:32,293 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:16:32,294 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,294 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:16:32,294 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,294 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,295 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,296 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:16:32,296 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 969e097efa67bfc5f2c39f5533a48c02 columnFamilyName A 2024-11-27T00:16:32,296 DEBUG [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:32,297 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.HStore(327): Store=969e097efa67bfc5f2c39f5533a48c02/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:16:32,297 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,298 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:16:32,298 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 969e097efa67bfc5f2c39f5533a48c02 columnFamilyName B 2024-11-27T00:16:32,298 DEBUG [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:32,298 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.HStore(327): Store=969e097efa67bfc5f2c39f5533a48c02/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:16:32,298 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,299 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:16:32,299 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 969e097efa67bfc5f2c39f5533a48c02 columnFamilyName C 2024-11-27T00:16:32,299 DEBUG [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:32,299 INFO [StoreOpener-969e097efa67bfc5f2c39f5533a48c02-1 {}] regionserver.HStore(327): Store=969e097efa67bfc5f2c39f5533a48c02/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:16:32,300 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:32,300 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,300 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,301 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,301 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,301 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,302 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:16:32,303 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,305 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T00:16:32,305 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 969e097efa67bfc5f2c39f5533a48c02; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63407446, jitterRate=-0.055155426263809204}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:16:32,305 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:32,306 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 969e097efa67bfc5f2c39f5533a48c02: Running coprocessor pre-open hook at 1732666592294Writing region info on filesystem at 1732666592294Initializing all the Stores at 1732666592295 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666592295Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666592295Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666592295Cleaning up temporary data from old regions at 1732666592301 (+6 ms)Running coprocessor post-open hooks at 1732666592305 (+4 ms)Region opened successfully at 1732666592306 (+1 ms) 2024-11-27T00:16:32,307 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., pid=122, masterSystemTime=1732666592290 2024-11-27T00:16:32,308 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:32,308 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:32,309 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=969e097efa67bfc5f2c39f5533a48c02, regionState=OPEN, openSeqNum=2, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:32,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:16:32,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-27T00:16:32,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 in 173 msec 2024-11-27T00:16:32,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-27T00:16:32,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, ASSIGN in 332 msec 2024-11-27T00:16:32,315 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T00:16:32,315 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666592315"}]},"ts":"1732666592315"} 2024-11-27T00:16:32,317 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T00:16:32,318 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T00:16:32,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-27T00:16:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-27T00:16:33,274 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:33,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,278 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f458428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,280 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40df5397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,281 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,284 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T00:16:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is f036bc4986ea,33599,1732666453115 2024-11-27T00:16:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1da04185 2024-11-27T00:16:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T00:16:33,286 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T00:16:33,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,288 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d4dd7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,290 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3c87d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,296 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b9a27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,298 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12594543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf5c9e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,301 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ebc59e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,303 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea9d8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,305 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58782142, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,306 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32bbc1b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,307 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2712d793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,309 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a0af2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,310 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ac33f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,311 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1372055b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,313 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c754bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233f05da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,315 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce343bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,317 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd5982a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,318 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17875a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:33,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:33,320 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b938fc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:33,321 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb31130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:33,322 DEBUG [Thread-1958 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,322 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,322 DEBUG [Thread-1951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,322 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,322 DEBUG [Thread-1960 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,322 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,323 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,323 DEBUG [Thread-1960 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,323 DEBUG [Thread-1954 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,323 DEBUG [Thread-1960 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,324 DEBUG [Thread-1962 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,324 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,325 DEBUG [Thread-1956 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:33,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:33,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:33,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2] 2024-11-27T00:16:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-27T00:16:33,332 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T00:16:33,333 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:33,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:33,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T00:16:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:33,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/2eca29fd10914d08adfa2feabe787ce6 is 50, key is test_row_0/A:col10/1732666593332/Put/seqid=0 2024-11-27T00:16:33,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742256_1432 (size=16681) 2024-11-27T00:16:33,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/2eca29fd10914d08adfa2feabe787ce6 2024-11-27T00:16:33,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666593353, max timestamp=1732666593372Num uniques 859; 2024-11-27T00:16:33,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666593353, max timestamp=1732666593372Num uniques 859; 2024-11-27T00:16:33,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:33,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:33,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666593353, max timestamp=1732666593372Num uniques 859; 2024-11-27T00:16:33,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:33,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/69728658fa924293bc9868540a7c3fee is 50, key is test_row_0/B:col10/1732666593332/Put/seqid=0 2024-11-27T00:16:33,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742257_1433 (size=12001) 2024-11-27T00:16:33,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/69728658fa924293bc9868540a7c3fee 2024-11-27T00:16:33,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666653392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666653403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/869fd4cad8014bb2af6be8d4b4b78797 is 50, key is test_row_0/C:col10/1732666593332/Put/seqid=0 2024-11-27T00:16:33,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742258_1434 (size=12001) 2024-11-27T00:16:33,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666653404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666653404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666653405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T00:16:33,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T00:16:33,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:33,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:33,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:33,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666653514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666653514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666653524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666653524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666653524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T00:16:33,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:33,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:33,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:33,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T00:16:33,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666653724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666653724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666653743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666653744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666653744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:33,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T00:16:33,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:33,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:33,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:33,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:33,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/869fd4cad8014bb2af6be8d4b4b78797 2024-11-27T00:16:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/2eca29fd10914d08adfa2feabe787ce6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2eca29fd10914d08adfa2feabe787ce6 2024-11-27T00:16:33,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2eca29fd10914d08adfa2feabe787ce6, entries=250, sequenceid=12, filesize=16.3 K 2024-11-27T00:16:33,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/69728658fa924293bc9868540a7c3fee as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/69728658fa924293bc9868540a7c3fee 2024-11-27T00:16:33,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/69728658fa924293bc9868540a7c3fee, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T00:16:33,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/869fd4cad8014bb2af6be8d4b4b78797 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/869fd4cad8014bb2af6be8d4b4b78797 2024-11-27T00:16:33,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/869fd4cad8014bb2af6be8d4b4b78797, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T00:16:33,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 969e097efa67bfc5f2c39f5533a48c02 in 494ms, sequenceid=12, compaction requested=false 2024-11-27T00:16:33,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:33,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T00:16:33,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:33,945 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-27T00:16:33,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:33,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:33,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:33,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:33,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:33,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:33,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/9871121b889648aea17244aee8edfcc4 is 50, key is test_row_0/A:col10/1732666593403/Put/seqid=0 2024-11-27T00:16:33,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742259_1435 (size=12001) 2024-11-27T00:16:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T00:16:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:34,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:34,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666654064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666654064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666654065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666654066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666654066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666654185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666654185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666654185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666654185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666654185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,354 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/9871121b889648aea17244aee8edfcc4 2024-11-27T00:16:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/88f69103eb01413f95299f58d283035f is 50, key is test_row_0/B:col10/1732666593403/Put/seqid=0 2024-11-27T00:16:34,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742260_1436 (size=12001) 2024-11-27T00:16:34,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666654403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666654403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666654403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666654404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666654404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T00:16:34,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666654714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666654714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666654714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666654714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666654715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:34,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:34,763 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/88f69103eb01413f95299f58d283035f 2024-11-27T00:16:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/efa1f1f3f6a94c438f54b998631b4af9 is 50, key is test_row_0/C:col10/1732666593403/Put/seqid=0 2024-11-27T00:16:34,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742261_1437 (size=12001) 2024-11-27T00:16:35,175 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/efa1f1f3f6a94c438f54b998631b4af9 2024-11-27T00:16:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/9871121b889648aea17244aee8edfcc4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9871121b889648aea17244aee8edfcc4 2024-11-27T00:16:35,186 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9871121b889648aea17244aee8edfcc4, entries=150, sequenceid=47, filesize=11.7 K 2024-11-27T00:16:35,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/88f69103eb01413f95299f58d283035f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/88f69103eb01413f95299f58d283035f 2024-11-27T00:16:35,190 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/88f69103eb01413f95299f58d283035f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-27T00:16:35,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/efa1f1f3f6a94c438f54b998631b4af9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/efa1f1f3f6a94c438f54b998631b4af9 2024-11-27T00:16:35,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/efa1f1f3f6a94c438f54b998631b4af9, entries=150, sequenceid=47, filesize=11.7 K 2024-11-27T00:16:35,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 969e097efa67bfc5f2c39f5533a48c02 in 1249ms, sequenceid=47, compaction requested=false 2024-11-27T00:16:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:35,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-27T00:16:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-27T00:16:35,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-27T00:16:35,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-11-27T00:16:35,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8710 sec 2024-11-27T00:16:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:35,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T00:16:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:35,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/0b6a5125af4442e9acccd97145d84272 is 50, key is test_row_0/A:col10/1732666594064/Put/seqid=0 2024-11-27T00:16:35,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742262_1438 (size=14341) 2024-11-27T00:16:35,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/0b6a5125af4442e9acccd97145d84272 2024-11-27T00:16:35,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/b424aacbec9640f6aeb0b907280e5749 is 50, key is test_row_0/B:col10/1732666594064/Put/seqid=0 2024-11-27T00:16:35,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742263_1439 (size=12001) 2024-11-27T00:16:35,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666655259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666655260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666655264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666655266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666655271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666655374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666655383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666655383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666655383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666655384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T00:16:35,484 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:35,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-27T00:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T00:16:35,486 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:35,486 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:35,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:35,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666655583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T00:16:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666655594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666655594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666655594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666655604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T00:16:35,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:35,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:35,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:35,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:35,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:35,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/b424aacbec9640f6aeb0b907280e5749 2024-11-27T00:16:35,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/4b34d351457d4da59d102a55946013a0 is 50, key is test_row_0/C:col10/1732666594064/Put/seqid=0 2024-11-27T00:16:35,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742264_1440 (size=12001) 2024-11-27T00:16:35,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T00:16:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:35,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:35,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:35,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T00:16:35,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666655893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666655904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666655904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666655904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666655914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:35,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:35,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T00:16:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:35,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:36,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/4b34d351457d4da59d102a55946013a0 2024-11-27T00:16:36,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/0b6a5125af4442e9acccd97145d84272 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/0b6a5125af4442e9acccd97145d84272 2024-11-27T00:16:36,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/0b6a5125af4442e9acccd97145d84272, entries=200, sequenceid=59, filesize=14.0 K 2024-11-27T00:16:36,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/b424aacbec9640f6aeb0b907280e5749 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b424aacbec9640f6aeb0b907280e5749 2024-11-27T00:16:36,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b424aacbec9640f6aeb0b907280e5749, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T00:16:36,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/4b34d351457d4da59d102a55946013a0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/4b34d351457d4da59d102a55946013a0 2024-11-27T00:16:36,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/4b34d351457d4da59d102a55946013a0, entries=150, sequenceid=59, filesize=11.7 K 2024-11-27T00:16:36,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 969e097efa67bfc5f2c39f5533a48c02 in 866ms, sequenceid=59, compaction requested=true 2024-11-27T00:16:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:36,094 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:36,094 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:36,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:36,095 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:36,095 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:36,095 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/A is initiating minor compaction (all files) 2024-11-27T00:16:36,095 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/B is initiating minor compaction (all files) 2024-11-27T00:16:36,095 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/B in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:36,095 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/A in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:36,095 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2eca29fd10914d08adfa2feabe787ce6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9871121b889648aea17244aee8edfcc4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/0b6a5125af4442e9acccd97145d84272] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=42.0 K 2024-11-27T00:16:36,095 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/69728658fa924293bc9868540a7c3fee, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/88f69103eb01413f95299f58d283035f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b424aacbec9640f6aeb0b907280e5749] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=35.2 K 2024-11-27T00:16:36,095 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eca29fd10914d08adfa2feabe787ce6, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732666593332 2024-11-27T00:16:36,095 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 69728658fa924293bc9868540a7c3fee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732666593332 2024-11-27T00:16:36,095 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f69103eb01413f95299f58d283035f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732666593389 2024-11-27T00:16:36,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T00:16:36,096 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9871121b889648aea17244aee8edfcc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732666593389 2024-11-27T00:16:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:36,096 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b424aacbec9640f6aeb0b907280e5749, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732666594063 2024-11-27T00:16:36,096 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b6a5125af4442e9acccd97145d84272, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732666594063 2024-11-27T00:16:36,096 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:16:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/21b3db6638f64e428c955686ed6e54bb is 50, key is test_row_0/A:col10/1732666595241/Put/seqid=0 2024-11-27T00:16:36,104 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#A#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:36,105 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/ec7be938f6f04a64a8fe83bf5c3018a4 is 50, key is test_row_0/A:col10/1732666594064/Put/seqid=0 2024-11-27T00:16:36,105 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#B#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:36,105 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/7deec6e08dcc4813aabec7606ed0876e is 50, key is test_row_0/B:col10/1732666594064/Put/seqid=0 2024-11-27T00:16:36,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742265_1441 (size=12001) 2024-11-27T00:16:36,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742266_1442 (size=12104) 2024-11-27T00:16:36,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742267_1443 (size=12104) 2024-11-27T00:16:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T00:16:36,117 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/7deec6e08dcc4813aabec7606ed0876e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7deec6e08dcc4813aabec7606ed0876e 2024-11-27T00:16:36,122 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/B of 969e097efa67bfc5f2c39f5533a48c02 into 7deec6e08dcc4813aabec7606ed0876e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:36,122 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:36,122 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/B, priority=13, startTime=1732666596094; duration=0sec 2024-11-27T00:16:36,122 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:36,122 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:B 2024-11-27T00:16:36,122 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:36,122 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:36,123 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/C is initiating minor compaction (all files) 2024-11-27T00:16:36,123 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/C in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:36,123 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/869fd4cad8014bb2af6be8d4b4b78797, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/efa1f1f3f6a94c438f54b998631b4af9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/4b34d351457d4da59d102a55946013a0] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=35.2 K 2024-11-27T00:16:36,123 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 869fd4cad8014bb2af6be8d4b4b78797, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732666593332 2024-11-27T00:16:36,123 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting efa1f1f3f6a94c438f54b998631b4af9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732666593389 2024-11-27T00:16:36,123 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b34d351457d4da59d102a55946013a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732666594063 2024-11-27T00:16:36,130 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:36,131 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/3d2f499d7bcf4b6f86aea2ca8e5e7b9a is 50, key is test_row_0/C:col10/1732666594064/Put/seqid=0 2024-11-27T00:16:36,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742268_1444 (size=12104) 2024-11-27T00:16:36,232 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T00:16:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:36,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:36,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666656428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666656433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666656436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666656436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666656437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,508 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/21b3db6638f64e428c955686ed6e54bb 2024-11-27T00:16:36,514 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/ec7be938f6f04a64a8fe83bf5c3018a4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ec7be938f6f04a64a8fe83bf5c3018a4 2024-11-27T00:16:36,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/9c2b0d4704064506b412b31d22418f12 is 50, key is test_row_0/B:col10/1732666595241/Put/seqid=0 2024-11-27T00:16:36,517 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/A of 969e097efa67bfc5f2c39f5533a48c02 into ec7be938f6f04a64a8fe83bf5c3018a4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:36,517 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:36,517 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/A, priority=13, startTime=1732666596094; duration=0sec 2024-11-27T00:16:36,518 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:36,518 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:A 2024-11-27T00:16:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742269_1445 (size=12001) 2024-11-27T00:16:36,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666656544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666656544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666656544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666656554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,577 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/3d2f499d7bcf4b6f86aea2ca8e5e7b9a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3d2f499d7bcf4b6f86aea2ca8e5e7b9a 2024-11-27T00:16:36,581 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/C of 969e097efa67bfc5f2c39f5533a48c02 into 3d2f499d7bcf4b6f86aea2ca8e5e7b9a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:36,581 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:36,581 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/C, priority=13, startTime=1732666596094; duration=0sec 2024-11-27T00:16:36,581 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:36,581 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:C 2024-11-27T00:16:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T00:16:36,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666656763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666656763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666656764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666656774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:36,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/9c2b0d4704064506b412b31d22418f12 2024-11-27T00:16:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/0ad3d4eee6a649b79c479b921e0df8ea is 50, key is test_row_0/C:col10/1732666595241/Put/seqid=0 2024-11-27T00:16:36,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742270_1446 (size=12001) 2024-11-27T00:16:37,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666657074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666657074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666657075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666657093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,335 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/0ad3d4eee6a649b79c479b921e0df8ea 2024-11-27T00:16:37,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/21b3db6638f64e428c955686ed6e54bb as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/21b3db6638f64e428c955686ed6e54bb 2024-11-27T00:16:37,342 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/21b3db6638f64e428c955686ed6e54bb, entries=150, sequenceid=79, filesize=11.7 K 2024-11-27T00:16:37,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/9c2b0d4704064506b412b31d22418f12 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/9c2b0d4704064506b412b31d22418f12 2024-11-27T00:16:37,345 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/9c2b0d4704064506b412b31d22418f12, entries=150, sequenceid=79, filesize=11.7 K 2024-11-27T00:16:37,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/0ad3d4eee6a649b79c479b921e0df8ea as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ad3d4eee6a649b79c479b921e0df8ea 2024-11-27T00:16:37,349 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ad3d4eee6a649b79c479b921e0df8ea, entries=150, sequenceid=79, filesize=11.7 K 2024-11-27T00:16:37,350 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 969e097efa67bfc5f2c39f5533a48c02 in 1254ms, sequenceid=79, compaction requested=false 2024-11-27T00:16:37,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:37,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:37,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-27T00:16:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-27T00:16:37,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-27T00:16:37,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8650 sec 2024-11-27T00:16:37,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8690 sec 2024-11-27T00:16:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:37,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:16:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:37,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:37,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/96ef72cfdfe34a0bac7f7ba38ca8af37 is 50, key is test_row_0/A:col10/1732666596435/Put/seqid=0 2024-11-27T00:16:37,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742271_1447 (size=14341) 2024-11-27T00:16:37,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666657576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666657584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666657593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666657593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666657614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T00:16:37,634 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:37,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-27T00:16:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T00:16:37,636 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:37,637 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:37,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:37,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666657694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T00:16:37,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T00:16:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:37,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:37,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/96ef72cfdfe34a0bac7f7ba38ca8af37 2024-11-27T00:16:37,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/1d8f297228b04a46bb3ae5964fcde1b1 is 50, key is test_row_0/B:col10/1732666596435/Put/seqid=0 2024-11-27T00:16:37,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742272_1448 (size=12001) 2024-11-27T00:16:37,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666657913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:37,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T00:16:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:37,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T00:16:38,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T00:16:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666658224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:38,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:38,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T00:16:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T00:16:38,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/1d8f297228b04a46bb3ae5964fcde1b1 2024-11-27T00:16:38,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/44ff29db89964309aa5f4d9d726e275f is 50, key is test_row_0/C:col10/1732666596435/Put/seqid=0 2024-11-27T00:16:38,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742273_1449 (size=12001) 2024-11-27T00:16:38,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T00:16:38,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:38,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T00:16:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:38,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666658594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:38,600 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:38,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666658614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:38,621 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:38,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666658623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:38,629 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:38,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666658634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:38,639 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:38,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/44ff29db89964309aa5f4d9d726e275f 2024-11-27T00:16:38,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/96ef72cfdfe34a0bac7f7ba38ca8af37 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/96ef72cfdfe34a0bac7f7ba38ca8af37 2024-11-27T00:16:38,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/96ef72cfdfe34a0bac7f7ba38ca8af37, entries=200, sequenceid=96, filesize=14.0 K 2024-11-27T00:16:38,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/1d8f297228b04a46bb3ae5964fcde1b1 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/1d8f297228b04a46bb3ae5964fcde1b1 2024-11-27T00:16:38,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/1d8f297228b04a46bb3ae5964fcde1b1, entries=150, sequenceid=96, filesize=11.7 K 2024-11-27T00:16:38,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/44ff29db89964309aa5f4d9d726e275f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/44ff29db89964309aa5f4d9d726e275f 2024-11-27T00:16:38,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/44ff29db89964309aa5f4d9d726e275f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-27T00:16:38,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 969e097efa67bfc5f2c39f5533a48c02 in 1242ms, sequenceid=96, compaction requested=true 2024-11-27T00:16:38,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:38,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:38,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:38,701 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:38,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:38,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:38,701 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:38,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:38,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:38,702 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:38,702 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:38,702 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/A is initiating minor compaction (all files) 2024-11-27T00:16:38,702 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/B is initiating minor compaction (all files) 2024-11-27T00:16:38,702 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/B in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,702 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/A in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,702 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ec7be938f6f04a64a8fe83bf5c3018a4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/21b3db6638f64e428c955686ed6e54bb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/96ef72cfdfe34a0bac7f7ba38ca8af37] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=37.5 K 2024-11-27T00:16:38,702 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7deec6e08dcc4813aabec7606ed0876e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/9c2b0d4704064506b412b31d22418f12, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/1d8f297228b04a46bb3ae5964fcde1b1] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=35.3 K 2024-11-27T00:16:38,702 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec7be938f6f04a64a8fe83bf5c3018a4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732666594063 2024-11-27T00:16:38,702 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7deec6e08dcc4813aabec7606ed0876e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732666594063 2024-11-27T00:16:38,703 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b3db6638f64e428c955686ed6e54bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732666595241 2024-11-27T00:16:38,703 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2b0d4704064506b412b31d22418f12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732666595241 2024-11-27T00:16:38,703 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96ef72cfdfe34a0bac7f7ba38ca8af37, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666596425 2024-11-27T00:16:38,703 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d8f297228b04a46bb3ae5964fcde1b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666596435 2024-11-27T00:16:38,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T00:16:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,705 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:16:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:38,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/de9dda19fceb4e70a7dba2293149074b is 50, key is test_row_0/A:col10/1732666597491/Put/seqid=0 2024-11-27T00:16:38,709 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:38,709 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#A#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:38,710 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/155368d6083c4a94907f817aeaeb039a is 50, key is test_row_0/B:col10/1732666596435/Put/seqid=0 2024-11-27T00:16:38,710 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/29b5e5b152c8488398f3852634d2a464 is 50, key is test_row_0/A:col10/1732666596435/Put/seqid=0 2024-11-27T00:16:38,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742274_1450 (size=12001) 2024-11-27T00:16:38,714 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/de9dda19fceb4e70a7dba2293149074b 2024-11-27T00:16:38,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742276_1452 (size=12207) 2024-11-27T00:16:38,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742275_1451 (size=12207) 2024-11-27T00:16:38,724 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/155368d6083c4a94907f817aeaeb039a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/155368d6083c4a94907f817aeaeb039a 2024-11-27T00:16:38,724 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/29b5e5b152c8488398f3852634d2a464 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/29b5e5b152c8488398f3852634d2a464 2024-11-27T00:16:38,732 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/A of 969e097efa67bfc5f2c39f5533a48c02 into 29b5e5b152c8488398f3852634d2a464(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:38,732 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:38,732 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/A, priority=13, startTime=1732666598701; duration=0sec 2024-11-27T00:16:38,732 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:38,732 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:A 2024-11-27T00:16:38,732 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:38,733 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:38,733 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/C is initiating minor compaction (all files) 2024-11-27T00:16:38,733 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/C in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:38,733 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3d2f499d7bcf4b6f86aea2ca8e5e7b9a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ad3d4eee6a649b79c479b921e0df8ea, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/44ff29db89964309aa5f4d9d726e275f] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=35.3 K 2024-11-27T00:16:38,733 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d2f499d7bcf4b6f86aea2ca8e5e7b9a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732666594063 2024-11-27T00:16:38,734 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad3d4eee6a649b79c479b921e0df8ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732666595241 2024-11-27T00:16:38,734 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/B of 969e097efa67bfc5f2c39f5533a48c02 into 155368d6083c4a94907f817aeaeb039a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:38,734 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:38,734 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/B, priority=13, startTime=1732666598701; duration=0sec 2024-11-27T00:16:38,734 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:38,734 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:B 2024-11-27T00:16:38,734 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ff29db89964309aa5f4d9d726e275f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666596435 2024-11-27T00:16:38,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/2103c67b48fe4cdabb7cc8fe403f97b2 is 50, key is test_row_0/B:col10/1732666597491/Put/seqid=0 2024-11-27T00:16:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742277_1453 (size=12001) 2024-11-27T00:16:38,742 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:38,742 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/146a627c8b6c479db0386095471319f4 is 50, key is test_row_0/C:col10/1732666596435/Put/seqid=0 2024-11-27T00:16:38,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742278_1454 (size=12207) 2024-11-27T00:16:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:38,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T00:16:38,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666658811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:38,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666658924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:39,141 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/2103c67b48fe4cdabb7cc8fe403f97b2 2024-11-27T00:16:39,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666659143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:39,154 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/146a627c8b6c479db0386095471319f4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/146a627c8b6c479db0386095471319f4 2024-11-27T00:16:39,160 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/C of 969e097efa67bfc5f2c39f5533a48c02 into 146a627c8b6c479db0386095471319f4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:39,160 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:39,161 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/C, priority=13, startTime=1732666598701; duration=0sec 2024-11-27T00:16:39,161 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:39,161 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:C 2024-11-27T00:16:39,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/0b044c5069394f75bc38dd5f3fe28718 is 50, key is test_row_0/C:col10/1732666597491/Put/seqid=0 2024-11-27T00:16:39,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742279_1455 (size=12001) 2024-11-27T00:16:39,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666659454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:39,568 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/0b044c5069394f75bc38dd5f3fe28718 2024-11-27T00:16:39,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/de9dda19fceb4e70a7dba2293149074b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/de9dda19fceb4e70a7dba2293149074b 2024-11-27T00:16:39,575 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/de9dda19fceb4e70a7dba2293149074b, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T00:16:39,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/2103c67b48fe4cdabb7cc8fe403f97b2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2103c67b48fe4cdabb7cc8fe403f97b2 2024-11-27T00:16:39,578 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2103c67b48fe4cdabb7cc8fe403f97b2, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T00:16:39,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/0b044c5069394f75bc38dd5f3fe28718 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0b044c5069394f75bc38dd5f3fe28718 2024-11-27T00:16:39,582 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0b044c5069394f75bc38dd5f3fe28718, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T00:16:39,583 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 969e097efa67bfc5f2c39f5533a48c02 in 878ms, sequenceid=114, compaction requested=false 2024-11-27T00:16:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-27T00:16:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-27T00:16:39,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-27T00:16:39,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9470 sec 2024-11-27T00:16:39,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.9510 sec 2024-11-27T00:16:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T00:16:39,784 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:39,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-27T00:16:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T00:16:39,786 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:39,787 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:39,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T00:16:39,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T00:16:39,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:39,939 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:16:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:39,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/230d7c20b3f046f3b64d63a3cc0253d6 is 50, key is test_row_0/A:col10/1732666598773/Put/seqid=0 2024-11-27T00:16:39,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742280_1456 (size=12051) 2024-11-27T00:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:39,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666660067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T00:16:40,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666660174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:40,348 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/230d7c20b3f046f3b64d63a3cc0253d6 2024-11-27T00:16:40,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/7d989eb6020e4d7184f4e94fbc55c9b7 is 50, key is test_row_0/B:col10/1732666598773/Put/seqid=0 2024-11-27T00:16:40,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742281_1457 (size=12051) 2024-11-27T00:16:40,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666660384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T00:16:40,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666660624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:40,628 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666660634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:40,638 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:40,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666660644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:40,650 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:40,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666660653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:40,660 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:40,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666660694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:40,757 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/7d989eb6020e4d7184f4e94fbc55c9b7 2024-11-27T00:16:40,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/5773bbf25f9d4e14a67acb885875a3e8 is 50, key is test_row_0/C:col10/1732666598773/Put/seqid=0 2024-11-27T00:16:40,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742282_1458 (size=12051) 2024-11-27T00:16:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T00:16:41,166 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/5773bbf25f9d4e14a67acb885875a3e8 2024-11-27T00:16:41,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/230d7c20b3f046f3b64d63a3cc0253d6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/230d7c20b3f046f3b64d63a3cc0253d6 2024-11-27T00:16:41,174 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/230d7c20b3f046f3b64d63a3cc0253d6, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T00:16:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/7d989eb6020e4d7184f4e94fbc55c9b7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7d989eb6020e4d7184f4e94fbc55c9b7 2024-11-27T00:16:41,178 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7d989eb6020e4d7184f4e94fbc55c9b7, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T00:16:41,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/5773bbf25f9d4e14a67acb885875a3e8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5773bbf25f9d4e14a67acb885875a3e8 2024-11-27T00:16:41,182 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5773bbf25f9d4e14a67acb885875a3e8, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T00:16:41,182 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 969e097efa67bfc5f2c39f5533a48c02 in 1243ms, sequenceid=131, compaction requested=true 2024-11-27T00:16:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-27T00:16:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-27T00:16:41,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-27T00:16:41,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-27T00:16:41,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4020 sec 2024-11-27T00:16:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:41,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:16:41,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:41,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:41,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:41,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:41,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:41,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:41,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/5edbb0b780b64227b2875c48336d9637 is 50, key is test_row_0/A:col10/1732666599991/Put/seqid=0 2024-11-27T00:16:41,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742283_1459 (size=14541) 2024-11-27T00:16:41,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666661315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:41,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666661424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:41,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:41,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666661633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:41,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/5edbb0b780b64227b2875c48336d9637 2024-11-27T00:16:41,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/a0b50fc3d23346feba1c8b0c24a002e2 is 50, key is test_row_0/B:col10/1732666599991/Put/seqid=0 2024-11-27T00:16:41,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742284_1460 (size=12151) 2024-11-27T00:16:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T00:16:41,934 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:41,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-27T00:16:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T00:16:41,937 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:41,937 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:41,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:41,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666661954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:41,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T00:16:42,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/a0b50fc3d23346feba1c8b0c24a002e2 2024-11-27T00:16:42,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/c13eb00434a349ae99d89879cbc88b85 is 50, key is test_row_0/C:col10/1732666599991/Put/seqid=0 2024-11-27T00:16:42,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742285_1461 (size=12151) 2024-11-27T00:16:42,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/c13eb00434a349ae99d89879cbc88b85 2024-11-27T00:16:42,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/5edbb0b780b64227b2875c48336d9637 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/5edbb0b780b64227b2875c48336d9637 2024-11-27T00:16:42,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/5edbb0b780b64227b2875c48336d9637, entries=200, sequenceid=150, filesize=14.2 K 2024-11-27T00:16:42,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/a0b50fc3d23346feba1c8b0c24a002e2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/a0b50fc3d23346feba1c8b0c24a002e2 2024-11-27T00:16:42,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/a0b50fc3d23346feba1c8b0c24a002e2, entries=150, sequenceid=150, filesize=11.9 K 2024-11-27T00:16:42,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/c13eb00434a349ae99d89879cbc88b85 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c13eb00434a349ae99d89879cbc88b85 2024-11-27T00:16:42,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c13eb00434a349ae99d89879cbc88b85, entries=150, sequenceid=150, filesize=11.9 K 2024-11-27T00:16:42,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 969e097efa67bfc5f2c39f5533a48c02 in 869ms, sequenceid=150, compaction requested=true 2024-11-27T00:16:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:42,082 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:42,082 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:42,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:42,083 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:42,083 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/A is initiating minor compaction (all files) 2024-11-27T00:16:42,083 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/A in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:42,083 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/29b5e5b152c8488398f3852634d2a464, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/de9dda19fceb4e70a7dba2293149074b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/230d7c20b3f046f3b64d63a3cc0253d6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/5edbb0b780b64227b2875c48336d9637] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=49.6 K 2024-11-27T00:16:42,084 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:42,084 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/B is initiating minor compaction (all files) 2024-11-27T00:16:42,084 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b5e5b152c8488398f3852634d2a464, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666596435 2024-11-27T00:16:42,084 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/B in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:42,084 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/155368d6083c4a94907f817aeaeb039a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2103c67b48fe4cdabb7cc8fe403f97b2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7d989eb6020e4d7184f4e94fbc55c9b7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/a0b50fc3d23346feba1c8b0c24a002e2] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=47.3 K 2024-11-27T00:16:42,084 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting de9dda19fceb4e70a7dba2293149074b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732666597491 2024-11-27T00:16:42,084 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 155368d6083c4a94907f817aeaeb039a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666596435 2024-11-27T00:16:42,084 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 230d7c20b3f046f3b64d63a3cc0253d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732666598773 2024-11-27T00:16:42,084 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2103c67b48fe4cdabb7cc8fe403f97b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732666597491 2024-11-27T00:16:42,084 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5edbb0b780b64227b2875c48336d9637, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732666599991 2024-11-27T00:16:42,085 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d989eb6020e4d7184f4e94fbc55c9b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732666598773 2024-11-27T00:16:42,085 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting a0b50fc3d23346feba1c8b0c24a002e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732666599991 2024-11-27T00:16:42,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T00:16:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:42,088 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T00:16:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/760216f48405485981d0bb52096b42b5 is 50, key is test_row_0/A:col10/1732666601290/Put/seqid=0 2024-11-27T00:16:42,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742286_1462 (size=12151) 2024-11-27T00:16:42,098 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#B#compaction#393 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:42,098 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/2929daca9c704f7e9aebe6811d4f6f96 is 50, key is test_row_0/B:col10/1732666599991/Put/seqid=0 2024-11-27T00:16:42,100 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:42,100 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/2124b94380f242ce9c6908f6aa62d527 is 50, key is test_row_0/A:col10/1732666599991/Put/seqid=0 2024-11-27T00:16:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742287_1463 (size=12493) 2024-11-27T00:16:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742288_1464 (size=12493) 2024-11-27T00:16:42,122 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T00:16:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T00:16:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:42,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:42,497 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/760216f48405485981d0bb52096b42b5 2024-11-27T00:16:42,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/4d8cc3ce4c2649de8f34b0ab05b55f8b is 50, key is test_row_0/B:col10/1732666601290/Put/seqid=0 2024-11-27T00:16:42,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742289_1465 (size=12151) 2024-11-27T00:16:42,512 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/2124b94380f242ce9c6908f6aa62d527 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2124b94380f242ce9c6908f6aa62d527 2024-11-27T00:16:42,512 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/2929daca9c704f7e9aebe6811d4f6f96 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2929daca9c704f7e9aebe6811d4f6f96 2024-11-27T00:16:42,522 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/B of 969e097efa67bfc5f2c39f5533a48c02 into 2929daca9c704f7e9aebe6811d4f6f96(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:42,522 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:42,522 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/B, priority=12, startTime=1732666602082; duration=0sec 2024-11-27T00:16:42,522 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:42,522 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/A of 969e097efa67bfc5f2c39f5533a48c02 into 2124b94380f242ce9c6908f6aa62d527(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:42,522 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:B 2024-11-27T00:16:42,522 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:42,522 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:42,522 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/A, priority=12, startTime=1732666602082; duration=0sec 2024-11-27T00:16:42,523 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:42,523 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:A 2024-11-27T00:16:42,523 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:42,523 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/C is initiating minor compaction (all files) 2024-11-27T00:16:42,523 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/C in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:42,523 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/146a627c8b6c479db0386095471319f4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0b044c5069394f75bc38dd5f3fe28718, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5773bbf25f9d4e14a67acb885875a3e8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c13eb00434a349ae99d89879cbc88b85] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=47.3 K 2024-11-27T00:16:42,524 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 146a627c8b6c479db0386095471319f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666596435 2024-11-27T00:16:42,524 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b044c5069394f75bc38dd5f3fe28718, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732666597491 2024-11-27T00:16:42,524 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5773bbf25f9d4e14a67acb885875a3e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732666598773 2024-11-27T00:16:42,524 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c13eb00434a349ae99d89879cbc88b85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732666599991 2024-11-27T00:16:42,533 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:42,533 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/0ee72df2b3ac414b8b60bd864dc3b0f6 is 50, key is test_row_0/C:col10/1732666599991/Put/seqid=0 2024-11-27T00:16:42,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742290_1466 (size=12493) 2024-11-27T00:16:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T00:16:42,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666662588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:42,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666662704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:42,909 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/4d8cc3ce4c2649de8f34b0ab05b55f8b 2024-11-27T00:16:42,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/668a2a7f393f4bc898ebbcb7f953a078 is 50, key is test_row_0/C:col10/1732666601290/Put/seqid=0 2024-11-27T00:16:42,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742291_1467 (size=12151) 2024-11-27T00:16:42,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666662924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:42,940 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/0ee72df2b3ac414b8b60bd864dc3b0f6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ee72df2b3ac414b8b60bd864dc3b0f6 2024-11-27T00:16:42,944 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/C of 969e097efa67bfc5f2c39f5533a48c02 into 0ee72df2b3ac414b8b60bd864dc3b0f6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:42,944 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:42,944 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/C, priority=12, startTime=1732666602082; duration=0sec 2024-11-27T00:16:42,944 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:42,944 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:C 2024-11-27T00:16:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T00:16:43,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666663233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:43,319 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/668a2a7f393f4bc898ebbcb7f953a078 2024-11-27T00:16:43,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/760216f48405485981d0bb52096b42b5 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/760216f48405485981d0bb52096b42b5 2024-11-27T00:16:43,325 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/760216f48405485981d0bb52096b42b5, entries=150, sequenceid=163, filesize=11.9 K 2024-11-27T00:16:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/4d8cc3ce4c2649de8f34b0ab05b55f8b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/4d8cc3ce4c2649de8f34b0ab05b55f8b 2024-11-27T00:16:43,328 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/4d8cc3ce4c2649de8f34b0ab05b55f8b, entries=150, sequenceid=163, filesize=11.9 K 2024-11-27T00:16:43,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/668a2a7f393f4bc898ebbcb7f953a078 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/668a2a7f393f4bc898ebbcb7f953a078 2024-11-27T00:16:43,332 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/668a2a7f393f4bc898ebbcb7f953a078, entries=150, sequenceid=163, filesize=11.9 K 2024-11-27T00:16:43,332 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 969e097efa67bfc5f2c39f5533a48c02 in 1244ms, sequenceid=163, compaction requested=false 2024-11-27T00:16:43,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:43,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:43,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-27T00:16:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-27T00:16:43,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-27T00:16:43,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3970 sec 2024-11-27T00:16:43,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4010 sec 2024-11-27T00:16:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:43,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:16:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:43,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/88f1083cdf764aa78282d46f0765544a is 50, key is test_row_0/A:col10/1732666602580/Put/seqid=0 2024-11-27T00:16:43,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742292_1468 (size=14541) 2024-11-27T00:16:43,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666663801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:43,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666663913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:43,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:43,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:43,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T00:16:44,094 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:44,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-27T00:16:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T00:16:44,096 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:44,097 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:44,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:44,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666664134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/88f1083cdf764aa78282d46f0765544a 2024-11-27T00:16:44,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/afb24c1b13a34312afbc6d035d656591 is 50, key is test_row_0/B:col10/1732666602580/Put/seqid=0 2024-11-27T00:16:44,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742293_1469 (size=12151) 2024-11-27T00:16:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T00:16:44,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:44,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:44,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:44,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:44,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T00:16:44,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666664454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:44,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:44,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/afb24c1b13a34312afbc6d035d656591 2024-11-27T00:16:44,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/52d4a889ab2f45f8a504ff004116e6e2 is 50, key is test_row_0/C:col10/1732666602580/Put/seqid=0 2024-11-27T00:16:44,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742294_1470 (size=12151) 2024-11-27T00:16:44,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/52d4a889ab2f45f8a504ff004116e6e2 2024-11-27T00:16:44,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/88f1083cdf764aa78282d46f0765544a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/88f1083cdf764aa78282d46f0765544a 2024-11-27T00:16:44,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/88f1083cdf764aa78282d46f0765544a, entries=200, sequenceid=186, filesize=14.2 K 2024-11-27T00:16:44,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/afb24c1b13a34312afbc6d035d656591 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/afb24c1b13a34312afbc6d035d656591 2024-11-27T00:16:44,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/afb24c1b13a34312afbc6d035d656591, entries=150, sequenceid=186, filesize=11.9 K 2024-11-27T00:16:44,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/52d4a889ab2f45f8a504ff004116e6e2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/52d4a889ab2f45f8a504ff004116e6e2 2024-11-27T00:16:44,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/52d4a889ab2f45f8a504ff004116e6e2, entries=150, sequenceid=186, filesize=11.9 K 2024-11-27T00:16:44,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 969e097efa67bfc5f2c39f5533a48c02 in 853ms, sequenceid=186, compaction requested=true 2024-11-27T00:16:44,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:44,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:44,600 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:44,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:44,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:44,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:44,600 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:44,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:44,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:44,600 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:44,601 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/A is initiating minor compaction (all files) 2024-11-27T00:16:44,601 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/A in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,601 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:44,601 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/B is initiating minor compaction (all files) 2024-11-27T00:16:44,601 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2124b94380f242ce9c6908f6aa62d527, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/760216f48405485981d0bb52096b42b5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/88f1083cdf764aa78282d46f0765544a] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=38.3 K 2024-11-27T00:16:44,601 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/B in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,601 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2929daca9c704f7e9aebe6811d4f6f96, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/4d8cc3ce4c2649de8f34b0ab05b55f8b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/afb24c1b13a34312afbc6d035d656591] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=35.9 K 2024-11-27T00:16:44,601 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2124b94380f242ce9c6908f6aa62d527, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732666599991 2024-11-27T00:16:44,601 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2929daca9c704f7e9aebe6811d4f6f96, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732666599991 2024-11-27T00:16:44,601 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 760216f48405485981d0bb52096b42b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732666601290 2024-11-27T00:16:44,601 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8cc3ce4c2649de8f34b0ab05b55f8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732666601290 2024-11-27T00:16:44,601 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f1083cdf764aa78282d46f0765544a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732666602566 2024-11-27T00:16:44,602 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting afb24c1b13a34312afbc6d035d656591, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732666602566 2024-11-27T00:16:44,610 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:44,610 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#A#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:44,610 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/c383194c1f504db998d9c4974b0d4458 is 50, key is test_row_0/B:col10/1732666602580/Put/seqid=0 2024-11-27T00:16:44,610 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/f291e3d49e8440f285c37e10cff497f6 is 50, key is test_row_0/A:col10/1732666602580/Put/seqid=0 2024-11-27T00:16:44,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742295_1471 (size=12595) 2024-11-27T00:16:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742296_1472 (size=12595) 2024-11-27T00:16:44,621 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/f291e3d49e8440f285c37e10cff497f6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f291e3d49e8440f285c37e10cff497f6 2024-11-27T00:16:44,624 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/A of 969e097efa67bfc5f2c39f5533a48c02 into f291e3d49e8440f285c37e10cff497f6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:44,624 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:44,624 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/A, priority=13, startTime=1732666604600; duration=0sec 2024-11-27T00:16:44,624 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:44,624 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:A 2024-11-27T00:16:44,624 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:44,625 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:44,625 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/C is initiating minor compaction (all files) 2024-11-27T00:16:44,625 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/C in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,625 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ee72df2b3ac414b8b60bd864dc3b0f6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/668a2a7f393f4bc898ebbcb7f953a078, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/52d4a889ab2f45f8a504ff004116e6e2] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=35.9 K 2024-11-27T00:16:44,625 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee72df2b3ac414b8b60bd864dc3b0f6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732666599991 2024-11-27T00:16:44,626 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 668a2a7f393f4bc898ebbcb7f953a078, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732666601290 2024-11-27T00:16:44,626 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52d4a889ab2f45f8a504ff004116e6e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732666602566 2024-11-27T00:16:44,630 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:44,631 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/686dd073304d4ea09da2f8da02612be8 is 50, key is test_row_0/C:col10/1732666602580/Put/seqid=0 2024-11-27T00:16:44,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742297_1473 (size=12595) 2024-11-27T00:16:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:44,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T00:16:44,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:44,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:44,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:44,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:44,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:44,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:44,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/e8830368c47d46988136e1720c197b4f is 50, key is test_row_0/A:col10/1732666603783/Put/seqid=0 2024-11-27T00:16:44,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742298_1474 (size=14541) 2024-11-27T00:16:44,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666664701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666664703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666664705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666664706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T00:16:44,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666664814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666664824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666664824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666664824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:44,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:44,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:44,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666664973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,025 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/c383194c1f504db998d9c4974b0d4458 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c383194c1f504db998d9c4974b0d4458 2024-11-27T00:16:45,029 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/B of 969e097efa67bfc5f2c39f5533a48c02 into c383194c1f504db998d9c4974b0d4458(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:45,029 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:45,029 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/B, priority=13, startTime=1732666604600; duration=0sec 2024-11-27T00:16:45,029 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:45,029 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:B 2024-11-27T00:16:45,038 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/686dd073304d4ea09da2f8da02612be8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/686dd073304d4ea09da2f8da02612be8 2024-11-27T00:16:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666665034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666665034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666665034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,042 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/C of 969e097efa67bfc5f2c39f5533a48c02 into 686dd073304d4ea09da2f8da02612be8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:45,042 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,042 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/C, priority=13, startTime=1732666604600; duration=0sec 2024-11-27T00:16:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666665035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,042 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:45,042 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:C 2024-11-27T00:16:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/e8830368c47d46988136e1720c197b4f 2024-11-27T00:16:45,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/59efad090edd43af9b8f0e11e6f89302 is 50, key is test_row_0/B:col10/1732666603783/Put/seqid=0 2024-11-27T00:16:45,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742299_1475 (size=12151) 2024-11-27T00:16:45,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/59efad090edd43af9b8f0e11e6f89302 2024-11-27T00:16:45,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/804f07685ff34a60afb40a44615620e4 is 50, key is test_row_0/C:col10/1732666603783/Put/seqid=0 2024-11-27T00:16:45,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742300_1476 (size=12151) 2024-11-27T00:16:45,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T00:16:45,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666665343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666665343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666665343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666665354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:45,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/804f07685ff34a60afb40a44615620e4 2024-11-27T00:16:45,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/e8830368c47d46988136e1720c197b4f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/e8830368c47d46988136e1720c197b4f 2024-11-27T00:16:45,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/e8830368c47d46988136e1720c197b4f, entries=200, sequenceid=200, filesize=14.2 K 2024-11-27T00:16:45,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/59efad090edd43af9b8f0e11e6f89302 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/59efad090edd43af9b8f0e11e6f89302 2024-11-27T00:16:45,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/59efad090edd43af9b8f0e11e6f89302, entries=150, sequenceid=200, filesize=11.9 K 2024-11-27T00:16:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/804f07685ff34a60afb40a44615620e4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/804f07685ff34a60afb40a44615620e4 2024-11-27T00:16:45,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/804f07685ff34a60afb40a44615620e4, entries=150, sequenceid=200, filesize=11.9 K 2024-11-27T00:16:45,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 969e097efa67bfc5f2c39f5533a48c02 in 857ms, sequenceid=200, compaction requested=false 2024-11-27T00:16:45,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:45,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T00:16:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:45,623 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:16:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/9959fd437120412097cec815bac4f797 is 50, key is test_row_0/A:col10/1732666604704/Put/seqid=0 2024-11-27T00:16:45,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742301_1477 (size=12151) 2024-11-27T00:16:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:45,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:45,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666665874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666665874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666665879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666665880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:45,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666665983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666665994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666665994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666665994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,001 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,032 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/9959fd437120412097cec815bac4f797 2024-11-27T00:16:46,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/93a7af8181ba4a1f96741b59bea69331 is 50, key is test_row_0/B:col10/1732666604704/Put/seqid=0 2024-11-27T00:16:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742302_1478 (size=12151) 2024-11-27T00:16:46,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666666193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666666204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666666204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T00:16:46,442 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/93a7af8181ba4a1f96741b59bea69331 2024-11-27T00:16:46,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/3e9d099533f84d00835b2f26e1ea636a is 50, key is test_row_0/C:col10/1732666604704/Put/seqid=0 2024-11-27T00:16:46,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742303_1479 (size=12151) 2024-11-27T00:16:46,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666666514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666666523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666666524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:46,851 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/3e9d099533f84d00835b2f26e1ea636a 2024-11-27T00:16:46,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/9959fd437120412097cec815bac4f797 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9959fd437120412097cec815bac4f797 2024-11-27T00:16:46,857 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9959fd437120412097cec815bac4f797, entries=150, sequenceid=221, filesize=11.9 K 2024-11-27T00:16:46,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/93a7af8181ba4a1f96741b59bea69331 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/93a7af8181ba4a1f96741b59bea69331 2024-11-27T00:16:46,861 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/93a7af8181ba4a1f96741b59bea69331, entries=150, sequenceid=221, filesize=11.9 K 2024-11-27T00:16:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/3e9d099533f84d00835b2f26e1ea636a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3e9d099533f84d00835b2f26e1ea636a 2024-11-27T00:16:46,864 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3e9d099533f84d00835b2f26e1ea636a, entries=150, sequenceid=221, filesize=11.9 K 2024-11-27T00:16:46,864 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 969e097efa67bfc5f2c39f5533a48c02 in 1241ms, sequenceid=221, compaction requested=true 2024-11-27T00:16:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-27T00:16:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-27T00:16:46,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-27T00:16:46,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7690 sec 2024-11-27T00:16:46,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.7730 sec 2024-11-27T00:16:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:46,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:16:46,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:46,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:46,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:46,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:46,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:46,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:46,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/c0257f72758b433aac1fb09c3ee3de5b is 50, key is test_row_0/A:col10/1732666606898/Put/seqid=0 2024-11-27T00:16:46,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742304_1480 (size=14541) 2024-11-27T00:16:47,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666667023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:47,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666667033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666667034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:47,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666667034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:47,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666667144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:47,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/c0257f72758b433aac1fb09c3ee3de5b 2024-11-27T00:16:47,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/b27f592446c6443fa32fbed8d6d70124 is 50, key is test_row_0/B:col10/1732666606898/Put/seqid=0 2024-11-27T00:16:47,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742305_1481 (size=12151) 2024-11-27T00:16:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666667354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666667663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:47,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/b27f592446c6443fa32fbed8d6d70124 2024-11-27T00:16:47,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/936df0d627e34cebbb2401db9ef241f9 is 50, key is test_row_0/C:col10/1732666606898/Put/seqid=0 2024-11-27T00:16:47,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742306_1482 (size=12151) 2024-11-27T00:16:48,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666668014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:48,028 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666668054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666668054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666668054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:48,066 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,066 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:48,067 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:48,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/936df0d627e34cebbb2401db9ef241f9 2024-11-27T00:16:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/c0257f72758b433aac1fb09c3ee3de5b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/c0257f72758b433aac1fb09c3ee3de5b 2024-11-27T00:16:48,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/c0257f72758b433aac1fb09c3ee3de5b, entries=200, sequenceid=235, filesize=14.2 K 2024-11-27T00:16:48,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/b27f592446c6443fa32fbed8d6d70124 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b27f592446c6443fa32fbed8d6d70124 2024-11-27T00:16:48,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b27f592446c6443fa32fbed8d6d70124, entries=150, sequenceid=235, filesize=11.9 K 2024-11-27T00:16:48,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/936df0d627e34cebbb2401db9ef241f9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/936df0d627e34cebbb2401db9ef241f9 2024-11-27T00:16:48,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/936df0d627e34cebbb2401db9ef241f9, entries=150, sequenceid=235, filesize=11.9 K 2024-11-27T00:16:48,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 969e097efa67bfc5f2c39f5533a48c02 in 1243ms, sequenceid=235, compaction requested=true 2024-11-27T00:16:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:48,142 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:48,142 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:48,143 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:48,143 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:48,143 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/A is initiating minor compaction (all files) 2024-11-27T00:16:48,143 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/B is initiating minor compaction (all files) 2024-11-27T00:16:48,143 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/B in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,143 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/A in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,143 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c383194c1f504db998d9c4974b0d4458, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/59efad090edd43af9b8f0e11e6f89302, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/93a7af8181ba4a1f96741b59bea69331, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b27f592446c6443fa32fbed8d6d70124] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=47.9 K 2024-11-27T00:16:48,143 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f291e3d49e8440f285c37e10cff497f6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/e8830368c47d46988136e1720c197b4f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9959fd437120412097cec815bac4f797, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/c0257f72758b433aac1fb09c3ee3de5b] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=52.6 K 2024-11-27T00:16:48,143 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c383194c1f504db998d9c4974b0d4458, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732666602566 2024-11-27T00:16:48,143 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f291e3d49e8440f285c37e10cff497f6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732666602566 2024-11-27T00:16:48,144 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 59efad090edd43af9b8f0e11e6f89302, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732666603783 2024-11-27T00:16:48,144 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8830368c47d46988136e1720c197b4f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732666603770 2024-11-27T00:16:48,144 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a7af8181ba4a1f96741b59bea69331, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732666604700 2024-11-27T00:16:48,144 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9959fd437120412097cec815bac4f797, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732666604700 2024-11-27T00:16:48,144 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b27f592446c6443fa32fbed8d6d70124, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666605871 2024-11-27T00:16:48,144 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0257f72758b433aac1fb09c3ee3de5b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666605871 2024-11-27T00:16:48,152 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#A#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:48,152 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#B#compaction#414 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:48,152 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/ff85ac4fc50846cea214d1e63e79e3a3 is 50, key is test_row_0/A:col10/1732666606898/Put/seqid=0 2024-11-27T00:16:48,152 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/c0e80b437aba4014a919b5adc5f9bd73 is 50, key is test_row_0/B:col10/1732666606898/Put/seqid=0 2024-11-27T00:16:48,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742307_1483 (size=12731) 2024-11-27T00:16:48,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742308_1484 (size=12731) 2024-11-27T00:16:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:48,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:16:48,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:48,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:48,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:48,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:48,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:48,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:48,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/051082cb329e4f9dbd42e95818259b55 is 50, key is test_row_0/A:col10/1732666606978/Put/seqid=0 2024-11-27T00:16:48,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742309_1485 (size=14541) 2024-11-27T00:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T00:16:48,264 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:48,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-27T00:16:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T00:16:48,266 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:48,266 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:48,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:48,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666668261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T00:16:48,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666668374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:48,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T00:16:48,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:48,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,565 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/ff85ac4fc50846cea214d1e63e79e3a3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ff85ac4fc50846cea214d1e63e79e3a3 2024-11-27T00:16:48,569 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/A of 969e097efa67bfc5f2c39f5533a48c02 into ff85ac4fc50846cea214d1e63e79e3a3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:48,569 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:48,569 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/A, priority=12, startTime=1732666608142; duration=0sec 2024-11-27T00:16:48,570 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:48,570 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:A 2024-11-27T00:16:48,570 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:48,570 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/c0e80b437aba4014a919b5adc5f9bd73 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c0e80b437aba4014a919b5adc5f9bd73 2024-11-27T00:16:48,571 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:48,571 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/C is initiating minor compaction (all files) 2024-11-27T00:16:48,571 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/C in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,571 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/686dd073304d4ea09da2f8da02612be8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/804f07685ff34a60afb40a44615620e4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3e9d099533f84d00835b2f26e1ea636a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/936df0d627e34cebbb2401db9ef241f9] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=47.9 K 2024-11-27T00:16:48,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T00:16:48,571 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 686dd073304d4ea09da2f8da02612be8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732666602566 2024-11-27T00:16:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,572 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 804f07685ff34a60afb40a44615620e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732666603783 2024-11-27T00:16:48,572 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e9d099533f84d00835b2f26e1ea636a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732666604700 2024-11-27T00:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,573 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936df0d627e34cebbb2401db9ef241f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666605871 2024-11-27T00:16:48,576 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/B of 969e097efa67bfc5f2c39f5533a48c02 into c0e80b437aba4014a919b5adc5f9bd73(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:48,576 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:48,577 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/B, priority=12, startTime=1732666608142; duration=0sec 2024-11-27T00:16:48,577 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:48,577 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:B 2024-11-27T00:16:48,581 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:48,582 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/676842bab2864077943f1a3a959fe98b is 50, key is test_row_0/C:col10/1732666606898/Put/seqid=0 2024-11-27T00:16:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T00:16:48,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742310_1486 (size=12731) 2024-11-27T00:16:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666668584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:48,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/051082cb329e4f9dbd42e95818259b55 2024-11-27T00:16:48,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/5be4710ba4fe42d49e818c3312860b26 is 50, key is test_row_0/B:col10/1732666606978/Put/seqid=0 2024-11-27T00:16:48,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742311_1487 (size=12151) 2024-11-27T00:16:48,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T00:16:48,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:48,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T00:16:48,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:48,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:48,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T00:16:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666668894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:48,989 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/676842bab2864077943f1a3a959fe98b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/676842bab2864077943f1a3a959fe98b 2024-11-27T00:16:48,992 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/C of 969e097efa67bfc5f2c39f5533a48c02 into 676842bab2864077943f1a3a959fe98b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:48,992 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:48,992 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/C, priority=12, startTime=1732666608142; duration=0sec 2024-11-27T00:16:48,992 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:48,992 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:C 2024-11-27T00:16:49,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/5be4710ba4fe42d49e818c3312860b26 2024-11-27T00:16:49,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/93c7a31155964e4bbb1b12fcf20f0213 is 50, key is test_row_0/C:col10/1732666606978/Put/seqid=0 2024-11-27T00:16:49,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742312_1488 (size=12151) 2024-11-27T00:16:49,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T00:16:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:49,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T00:16:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:49,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T00:16:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:49,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T00:16:49,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/93c7a31155964e4bbb1b12fcf20f0213 2024-11-27T00:16:49,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666669414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:49,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:49,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:49,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/051082cb329e4f9dbd42e95818259b55 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/051082cb329e4f9dbd42e95818259b55 2024-11-27T00:16:49,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/051082cb329e4f9dbd42e95818259b55, entries=200, sequenceid=254, filesize=14.2 K 2024-11-27T00:16:49,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/5be4710ba4fe42d49e818c3312860b26 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/5be4710ba4fe42d49e818c3312860b26 2024-11-27T00:16:49,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/5be4710ba4fe42d49e818c3312860b26, entries=150, sequenceid=254, filesize=11.9 K 2024-11-27T00:16:49,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/93c7a31155964e4bbb1b12fcf20f0213 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/93c7a31155964e4bbb1b12fcf20f0213 2024-11-27T00:16:49,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/93c7a31155964e4bbb1b12fcf20f0213, entries=150, sequenceid=254, filesize=11.9 K 2024-11-27T00:16:49,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 969e097efa67bfc5f2c39f5533a48c02 in 1241ms, sequenceid=254, compaction requested=false 2024-11-27T00:16:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:49,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T00:16:49,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:49,489 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T00:16:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:49,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/d5bc608b7f1e4e03a1d68e511c90ef10 is 50, key is test_row_0/A:col10/1732666608240/Put/seqid=0 2024-11-27T00:16:49,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742313_1489 (size=12301) 2024-11-27T00:16:49,497 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/d5bc608b7f1e4e03a1d68e511c90ef10 2024-11-27T00:16:49,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/aea6750850034dd5acb81a04900a3816 is 50, key is test_row_0/B:col10/1732666608240/Put/seqid=0 2024-11-27T00:16:49,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742314_1490 (size=12301) 2024-11-27T00:16:49,507 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/aea6750850034dd5acb81a04900a3816 2024-11-27T00:16:49,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/27ab760488f749e2bf49e502131f0a87 is 50, key is test_row_0/C:col10/1732666608240/Put/seqid=0 2024-11-27T00:16:49,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742315_1491 (size=12301) 2024-11-27T00:16:49,917 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/27ab760488f749e2bf49e502131f0a87 2024-11-27T00:16:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/d5bc608b7f1e4e03a1d68e511c90ef10 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d5bc608b7f1e4e03a1d68e511c90ef10 2024-11-27T00:16:49,923 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d5bc608b7f1e4e03a1d68e511c90ef10, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T00:16:49,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/aea6750850034dd5acb81a04900a3816 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/aea6750850034dd5acb81a04900a3816 2024-11-27T00:16:49,926 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/aea6750850034dd5acb81a04900a3816, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T00:16:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/27ab760488f749e2bf49e502131f0a87 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/27ab760488f749e2bf49e502131f0a87 2024-11-27T00:16:49,930 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/27ab760488f749e2bf49e502131f0a87, entries=150, sequenceid=270, filesize=12.0 K 2024-11-27T00:16:49,930 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 969e097efa67bfc5f2c39f5533a48c02 in 441ms, sequenceid=270, compaction requested=true 2024-11-27T00:16:49,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:49,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:49,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-27T00:16:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-27T00:16:49,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-27T00:16:49,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6660 sec 2024-11-27T00:16:49,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.6700 sec 2024-11-27T00:16:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:50,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:16:50,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:50,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:50,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:50,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:50,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:50,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:50,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/f2af2cc163ef462bb737ad4dcf670e30 is 50, key is test_row_0/A:col10/1732666610084/Put/seqid=0 2024-11-27T00:16:50,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742316_1492 (size=14741) 2024-11-27T00:16:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666670133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666610121, max timestamp=1732666610134Num uniques 859; 2024-11-27T00:16:50,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:50,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666610121, max timestamp=1732666610134Num uniques 859; 2024-11-27T00:16:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666610121, max timestamp=1732666610134Num uniques 859; 2024-11-27T00:16:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:16:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666670198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666670208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666670243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666670324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666670324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T00:16:50,424 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:50,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:50,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-27T00:16:50,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T00:16:50,426 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:50,427 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:50,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:50,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666670433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,441 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666670464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/f2af2cc163ef462bb737ad4dcf670e30 2024-11-27T00:16:50,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/d7918b27ab1d48dbb4287ca81a6d93ea is 50, key is test_row_0/B:col10/1732666610084/Put/seqid=0 2024-11-27T00:16:50,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742317_1493 (size=12301) 2024-11-27T00:16:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T00:16:50,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666670533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666670544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T00:16:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:50,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T00:16:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:50,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T00:16:50,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666670774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666670844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666670864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:50,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:50,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T00:16:50,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:50,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:50,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:50,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:50,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/d7918b27ab1d48dbb4287ca81a6d93ea 2024-11-27T00:16:50,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/5a904225322b4b679835d3b86605761b is 50, key is test_row_0/C:col10/1732666610084/Put/seqid=0 2024-11-27T00:16:50,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742318_1494 (size=12301) 2024-11-27T00:16:51,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T00:16:51,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:51,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:51,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T00:16:51,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T00:16:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:51,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666671294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:51,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/5a904225322b4b679835d3b86605761b 2024-11-27T00:16:51,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/f2af2cc163ef462bb737ad4dcf670e30 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f2af2cc163ef462bb737ad4dcf670e30 2024-11-27T00:16:51,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f2af2cc163ef462bb737ad4dcf670e30, entries=200, sequenceid=280, filesize=14.4 K 2024-11-27T00:16:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/d7918b27ab1d48dbb4287ca81a6d93ea as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/d7918b27ab1d48dbb4287ca81a6d93ea 2024-11-27T00:16:51,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/d7918b27ab1d48dbb4287ca81a6d93ea, entries=150, sequenceid=280, filesize=12.0 K 2024-11-27T00:16:51,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/5a904225322b4b679835d3b86605761b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5a904225322b4b679835d3b86605761b 2024-11-27T00:16:51,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5a904225322b4b679835d3b86605761b, entries=150, sequenceid=280, filesize=12.0 K 2024-11-27T00:16:51,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 969e097efa67bfc5f2c39f5533a48c02 in 1240ms, sequenceid=280, compaction requested=true 2024-11-27T00:16:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:51,327 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:51,327 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:51,332 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54314 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:51,332 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/A is initiating minor compaction (all files) 2024-11-27T00:16:51,332 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/A in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,332 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ff85ac4fc50846cea214d1e63e79e3a3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/051082cb329e4f9dbd42e95818259b55, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d5bc608b7f1e4e03a1d68e511c90ef10, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f2af2cc163ef462bb737ad4dcf670e30] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=53.0 K 2024-11-27T00:16:51,332 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff85ac4fc50846cea214d1e63e79e3a3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666605871 2024-11-27T00:16:51,332 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:51,332 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/B is initiating minor compaction (all files) 2024-11-27T00:16:51,333 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/B in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,333 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c0e80b437aba4014a919b5adc5f9bd73, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/5be4710ba4fe42d49e818c3312860b26, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/aea6750850034dd5acb81a04900a3816, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/d7918b27ab1d48dbb4287ca81a6d93ea] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=48.3 K 2024-11-27T00:16:51,333 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 051082cb329e4f9dbd42e95818259b55, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732666606978 2024-11-27T00:16:51,333 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c0e80b437aba4014a919b5adc5f9bd73, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666605871 2024-11-27T00:16:51,333 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5bc608b7f1e4e03a1d68e511c90ef10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732666608199 2024-11-27T00:16:51,333 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5be4710ba4fe42d49e818c3312860b26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732666606978 2024-11-27T00:16:51,334 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2af2cc163ef462bb737ad4dcf670e30, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732666610078 2024-11-27T00:16:51,334 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting aea6750850034dd5acb81a04900a3816, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732666608199 2024-11-27T00:16:51,334 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d7918b27ab1d48dbb4287ca81a6d93ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732666610079 2024-11-27T00:16:51,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-27T00:16:51,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,344 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T00:16:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:51,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:51,363 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:51,363 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/6190fbcdd78d4238b4bcc86b151b0966 is 50, key is test_row_0/A:col10/1732666610084/Put/seqid=0 2024-11-27T00:16:51,367 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#B#compaction#426 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:51,367 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/c81438d54677480e9ebb2f81946384f3 is 50, key is test_row_0/B:col10/1732666610084/Put/seqid=0 2024-11-27T00:16:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:51,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:51,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/87a46a3f32ae420e91d596a2b7e6f00c is 50, key is test_row_0/A:col10/1732666610197/Put/seqid=0 2024-11-27T00:16:51,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742320_1496 (size=13017) 2024-11-27T00:16:51,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742321_1497 (size=12301) 2024-11-27T00:16:51,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742319_1495 (size=13017) 2024-11-27T00:16:51,390 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/87a46a3f32ae420e91d596a2b7e6f00c 2024-11-27T00:16:51,392 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/c81438d54677480e9ebb2f81946384f3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c81438d54677480e9ebb2f81946384f3 2024-11-27T00:16:51,396 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/B of 969e097efa67bfc5f2c39f5533a48c02 into c81438d54677480e9ebb2f81946384f3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:51,396 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:51,397 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/B, priority=12, startTime=1732666611327; duration=0sec 2024-11-27T00:16:51,397 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:51,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/659de7da36094c2a9eebc1e12e1ce77e is 50, key is test_row_0/B:col10/1732666610197/Put/seqid=0 2024-11-27T00:16:51,397 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:B 2024-11-27T00:16:51,397 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T00:16:51,398 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T00:16:51,398 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/C is initiating minor compaction (all files) 2024-11-27T00:16:51,398 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/C in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,398 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/676842bab2864077943f1a3a959fe98b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/93c7a31155964e4bbb1b12fcf20f0213, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/27ab760488f749e2bf49e502131f0a87, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5a904225322b4b679835d3b86605761b] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=48.3 K 2024-11-27T00:16:51,398 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 676842bab2864077943f1a3a959fe98b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666605871 2024-11-27T00:16:51,399 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 93c7a31155964e4bbb1b12fcf20f0213, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732666606978 2024-11-27T00:16:51,399 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 27ab760488f749e2bf49e502131f0a87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732666608199 2024-11-27T00:16:51,399 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a904225322b4b679835d3b86605761b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732666610079 2024-11-27T00:16:51,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742322_1498 (size=12301) 2024-11-27T00:16:51,410 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/659de7da36094c2a9eebc1e12e1ce77e 2024-11-27T00:16:51,413 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:51,413 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/197e350bc3a041bcb049e8ebd816fd78 is 50, key is test_row_0/C:col10/1732666610084/Put/seqid=0 2024-11-27T00:16:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666671406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:51,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742323_1499 (size=13017) 2024-11-27T00:16:51,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/e60111b16b3344768872f9c4e2bd0351 is 50, key is test_row_0/C:col10/1732666610197/Put/seqid=0 2024-11-27T00:16:51,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666671419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:51,429 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/197e350bc3a041bcb049e8ebd816fd78 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/197e350bc3a041bcb049e8ebd816fd78 2024-11-27T00:16:51,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742324_1500 (size=12301) 2024-11-27T00:16:51,433 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/C of 969e097efa67bfc5f2c39f5533a48c02 into 197e350bc3a041bcb049e8ebd816fd78(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:51,433 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:51,433 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/C, priority=12, startTime=1732666611327; duration=0sec 2024-11-27T00:16:51,433 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:51,433 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:C 2024-11-27T00:16:51,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666671524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:51,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666671534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T00:16:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666671734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666671753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:51,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:51,830 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/e60111b16b3344768872f9c4e2bd0351 2024-11-27T00:16:51,839 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/6190fbcdd78d4238b4bcc86b151b0966 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/6190fbcdd78d4238b4bcc86b151b0966 2024-11-27T00:16:51,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/87a46a3f32ae420e91d596a2b7e6f00c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/87a46a3f32ae420e91d596a2b7e6f00c 2024-11-27T00:16:51,842 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/87a46a3f32ae420e91d596a2b7e6f00c, entries=150, sequenceid=316, filesize=12.0 K 2024-11-27T00:16:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/659de7da36094c2a9eebc1e12e1ce77e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/659de7da36094c2a9eebc1e12e1ce77e 2024-11-27T00:16:51,846 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/659de7da36094c2a9eebc1e12e1ce77e, entries=150, sequenceid=316, filesize=12.0 K 2024-11-27T00:16:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/e60111b16b3344768872f9c4e2bd0351 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/e60111b16b3344768872f9c4e2bd0351 2024-11-27T00:16:51,847 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/A of 969e097efa67bfc5f2c39f5533a48c02 into 6190fbcdd78d4238b4bcc86b151b0966(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:51,847 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:51,847 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/A, priority=12, startTime=1732666611327; duration=0sec 2024-11-27T00:16:51,847 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:51,847 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:A 2024-11-27T00:16:51,850 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/e60111b16b3344768872f9c4e2bd0351, entries=150, sequenceid=316, filesize=12.0 K 2024-11-27T00:16:51,851 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 969e097efa67bfc5f2c39f5533a48c02 in 507ms, sequenceid=316, compaction requested=false 2024-11-27T00:16:51,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:51,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:51,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-27T00:16:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-27T00:16:51,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-27T00:16:51,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4250 sec 2024-11-27T00:16:51,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4290 sec 2024-11-27T00:16:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:52,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T00:16:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:52,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:52,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/22560b70f3074c16a7d338ef9365978f is 50, key is test_row_0/A:col10/1732666612046/Put/seqid=0 2024-11-27T00:16:52,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742325_1501 (size=14741) 2024-11-27T00:16:52,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/22560b70f3074c16a7d338ef9365978f 2024-11-27T00:16:52,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/bd4466eb0ddf45d195ac5c201196557b is 50, key is test_row_0/B:col10/1732666612046/Put/seqid=0 2024-11-27T00:16:52,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742326_1502 (size=12301) 2024-11-27T00:16:52,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666672106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732666612090, max timestamp=1732666612114Num uniques 859; 2024-11-27T00:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732666612090, max timestamp=1732666612114Num uniques 859; 2024-11-27T00:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732666612090, max timestamp=1732666612114Num uniques 859; 2024-11-27T00:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T00:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T00:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-27T00:16:52,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666672174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666672175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666672223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666672294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666672294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47262 deadline: 1732666672313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,319 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666672433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/bd4466eb0ddf45d195ac5c201196557b 2024-11-27T00:16:52,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/b3ceda0138cc4afb802c0d89a57362cc is 50, key is test_row_0/C:col10/1732666612046/Put/seqid=0 2024-11-27T00:16:52,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742327_1503 (size=12301) 2024-11-27T00:16:52,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47286 deadline: 1732666672464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,473 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 969e097efa67bfc5f2c39f5533a48c02 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666672504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666672504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T00:16:52,574 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:52,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:16:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-27T00:16:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T00:16:52,576 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:16:52,577 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:16:52,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:16:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T00:16:52,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T00:16:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:52,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:52,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47280 deadline: 1732666672744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666672823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666672823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:52,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/b3ceda0138cc4afb802c0d89a57362cc 2024-11-27T00:16:52,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/22560b70f3074c16a7d338ef9365978f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/22560b70f3074c16a7d338ef9365978f 2024-11-27T00:16:52,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/22560b70f3074c16a7d338ef9365978f, entries=200, sequenceid=330, filesize=14.4 K 2024-11-27T00:16:52,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/bd4466eb0ddf45d195ac5c201196557b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/bd4466eb0ddf45d195ac5c201196557b 2024-11-27T00:16:52,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T00:16:52,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:52,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:52,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:52,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:52,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:52,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/bd4466eb0ddf45d195ac5c201196557b, entries=150, sequenceid=330, filesize=12.0 K 2024-11-27T00:16:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:16:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/b3ceda0138cc4afb802c0d89a57362cc as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3ceda0138cc4afb802c0d89a57362cc 2024-11-27T00:16:52,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3ceda0138cc4afb802c0d89a57362cc, entries=150, sequenceid=330, filesize=12.0 K 2024-11-27T00:16:52,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 969e097efa67bfc5f2c39f5533a48c02 in 839ms, sequenceid=330, compaction requested=true 2024-11-27T00:16:52,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:52,887 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:52,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:52,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:52,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:52,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:52,887 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:52,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:52,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:52,888 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:52,888 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:52,888 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/A is initiating minor compaction (all files) 2024-11-27T00:16:52,888 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/B is initiating minor compaction (all files) 2024-11-27T00:16:52,888 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/A in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:52,888 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/B in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:52,888 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/6190fbcdd78d4238b4bcc86b151b0966, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/87a46a3f32ae420e91d596a2b7e6f00c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/22560b70f3074c16a7d338ef9365978f] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=39.1 K 2024-11-27T00:16:52,888 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c81438d54677480e9ebb2f81946384f3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/659de7da36094c2a9eebc1e12e1ce77e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/bd4466eb0ddf45d195ac5c201196557b] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=36.7 K 2024-11-27T00:16:52,889 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6190fbcdd78d4238b4bcc86b151b0966, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732666610079 2024-11-27T00:16:52,889 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c81438d54677480e9ebb2f81946384f3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732666610079 2024-11-27T00:16:52,889 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87a46a3f32ae420e91d596a2b7e6f00c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732666610196 2024-11-27T00:16:52,889 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 659de7da36094c2a9eebc1e12e1ce77e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732666610196 2024-11-27T00:16:52,889 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting bd4466eb0ddf45d195ac5c201196557b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732666611405 2024-11-27T00:16:52,889 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22560b70f3074c16a7d338ef9365978f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732666611366 2024-11-27T00:16:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T00:16:52,895 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:52,895 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/a0318dc2f0744ed2afedbc2850d72901 is 50, key is test_row_0/A:col10/1732666612046/Put/seqid=0 2024-11-27T00:16:52,895 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:52,896 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/6e855e17fdf54898bdea19202ba490b4 is 50, key is test_row_0/B:col10/1732666612046/Put/seqid=0 2024-11-27T00:16:52,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742328_1504 (size=13119) 2024-11-27T00:16:52,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742329_1505 (size=13119) 2024-11-27T00:16:53,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-27T00:16:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:53,034 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-27T00:16:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:16:53,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/d4423bcda292422b974212a4e90b58ba is 50, key is test_row_0/A:col10/1732666612174/Put/seqid=0 2024-11-27T00:16:53,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742330_1506 (size=12301) 2024-11-27T00:16:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T00:16:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:53,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. as already flushing 2024-11-27T00:16:53,303 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/6e855e17fdf54898bdea19202ba490b4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/6e855e17fdf54898bdea19202ba490b4 2024-11-27T00:16:53,304 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/a0318dc2f0744ed2afedbc2850d72901 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/a0318dc2f0744ed2afedbc2850d72901 2024-11-27T00:16:53,308 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/B of 969e097efa67bfc5f2c39f5533a48c02 into 6e855e17fdf54898bdea19202ba490b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:53,308 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:53,308 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/B, priority=13, startTime=1732666612887; duration=0sec 2024-11-27T00:16:53,308 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:53,308 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:B 2024-11-27T00:16:53,309 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:53,309 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:53,310 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/C is initiating minor compaction (all files) 2024-11-27T00:16:53,310 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/C in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:53,310 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/197e350bc3a041bcb049e8ebd816fd78, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/e60111b16b3344768872f9c4e2bd0351, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3ceda0138cc4afb802c0d89a57362cc] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=36.7 K 2024-11-27T00:16:53,310 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 197e350bc3a041bcb049e8ebd816fd78, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732666610079 2024-11-27T00:16:53,310 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e60111b16b3344768872f9c4e2bd0351, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732666610196 2024-11-27T00:16:53,310 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ceda0138cc4afb802c0d89a57362cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732666611405 2024-11-27T00:16:53,318 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:53,319 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/7b2b6b7ef1784128a092a47c330c2ce7 is 50, key is test_row_0/C:col10/1732666612046/Put/seqid=0 2024-11-27T00:16:53,319 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/A of 969e097efa67bfc5f2c39f5533a48c02 into a0318dc2f0744ed2afedbc2850d72901(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:53,319 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:53,320 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/A, priority=13, startTime=1732666612887; duration=0sec 2024-11-27T00:16:53,320 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:53,320 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:A 2024-11-27T00:16:53,329 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-11-27T00:16:53,329 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:53,329 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,329 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:53,330 INFO [Thread-1954 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1954. 2024-11-27T00:16:53,330 DEBUG [Thread-1954 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:53,330 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,330 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:53,331 INFO [Thread-1956 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1956. 2024-11-27T00:16:53,331 DEBUG [Thread-1956 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:53,331 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,331 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:53,331 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-27T00:16:53,331 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:53,331 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,332 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:53,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742331_1507 (size=13119) 2024-11-27T00:16:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1732666673334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47258 deadline: 1732666673334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 , the old value is region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:16:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., hostname=f036bc4986ea,33881,1732666453890, seqNum=2 because the exception is null or not the one we care about 2024-11-27T00:16:53,341 INFO [Thread-1960 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1960. 2024-11-27T00:16:53,341 DEBUG [Thread-1960 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:53,341 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,341 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:53,346 INFO [Thread-1958 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1958. 2024-11-27T00:16:53,346 DEBUG [Thread-1958 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:53,346 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,346 DEBUG [Thread-1958 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:53,346 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:53,443 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/d4423bcda292422b974212a4e90b58ba 2024-11-27T00:16:53,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/7f1be266e176477c83b5fc32bda4013c is 50, key is test_row_0/B:col10/1732666612174/Put/seqid=0 2024-11-27T00:16:53,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742332_1508 (size=12301) 2024-11-27T00:16:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T00:16:53,740 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/7b2b6b7ef1784128a092a47c330c2ce7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/7b2b6b7ef1784128a092a47c330c2ce7 2024-11-27T00:16:53,744 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/C of 969e097efa67bfc5f2c39f5533a48c02 into 7b2b6b7ef1784128a092a47c330c2ce7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:53,744 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:53,744 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/C, priority=13, startTime=1732666612887; duration=0sec 2024-11-27T00:16:53,745 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:53,745 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:C 2024-11-27T00:16:53,856 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/7f1be266e176477c83b5fc32bda4013c 2024-11-27T00:16:53,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/353f2b3a8f9b431f8d81be7b4d6207f7 is 50, key is test_row_0/C:col10/1732666612174/Put/seqid=0 2024-11-27T00:16:53,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742333_1509 (size=12301) 2024-11-27T00:16:54,273 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/353f2b3a8f9b431f8d81be7b4d6207f7 2024-11-27T00:16:54,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/d4423bcda292422b974212a4e90b58ba as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d4423bcda292422b974212a4e90b58ba 2024-11-27T00:16:54,284 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d4423bcda292422b974212a4e90b58ba, entries=150, sequenceid=364, filesize=12.0 K 2024-11-27T00:16:54,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/7f1be266e176477c83b5fc32bda4013c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7f1be266e176477c83b5fc32bda4013c 2024-11-27T00:16:54,288 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7f1be266e176477c83b5fc32bda4013c, entries=150, sequenceid=364, filesize=12.0 K 2024-11-27T00:16:54,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/353f2b3a8f9b431f8d81be7b4d6207f7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/353f2b3a8f9b431f8d81be7b4d6207f7 2024-11-27T00:16:54,292 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/353f2b3a8f9b431f8d81be7b4d6207f7, entries=150, sequenceid=364, filesize=12.0 K 2024-11-27T00:16:54,293 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 969e097efa67bfc5f2c39f5533a48c02 in 1258ms, sequenceid=364, compaction requested=false 2024-11-27T00:16:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-27T00:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-27T00:16:54,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-27T00:16:54,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-11-27T00:16:54,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.7210 sec 2024-11-27T00:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:54,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-27T00:16:54,327 INFO [Thread-1951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1951. 2024-11-27T00:16:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:54,328 DEBUG [Thread-1951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:54,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:54,328 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:54,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:54,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:54,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:54,328 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:54,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:54,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:54,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/340d00fa720a4b9ebd57721c00edcf62 is 50, key is test_row_0/A:col10/1732666613295/Put/seqid=0 2024-11-27T00:16:54,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742334_1510 (size=12301) 2024-11-27T00:16:54,345 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-27T00:16:54,345 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-27T00:16:54,345 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:54,345 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:54,345 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:54,345 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:54,345 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:54,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:54,345 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:54,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-27T00:16:54,725 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:54,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/340d00fa720a4b9ebd57721c00edcf62 2024-11-27T00:16:54,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/8ae2049e1c444fe5ba4ae2cc106993b0 is 50, key is test_row_0/B:col10/1732666613295/Put/seqid=0 2024-11-27T00:16:54,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742335_1511 (size=12301) 2024-11-27T00:16:55,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/8ae2049e1c444fe5ba4ae2cc106993b0 2024-11-27T00:16:55,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/c99ea9886a2d49b89776572fd42e9ba6 is 50, key is test_row_0/C:col10/1732666613295/Put/seqid=0 2024-11-27T00:16:55,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742336_1512 (size=12301) 2024-11-27T00:16:55,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/c99ea9886a2d49b89776572fd42e9ba6 2024-11-27T00:16:55,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/340d00fa720a4b9ebd57721c00edcf62 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/340d00fa720a4b9ebd57721c00edcf62 2024-11-27T00:16:55,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/340d00fa720a4b9ebd57721c00edcf62, entries=150, sequenceid=379, filesize=12.0 K 2024-11-27T00:16:55,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/8ae2049e1c444fe5ba4ae2cc106993b0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/8ae2049e1c444fe5ba4ae2cc106993b0 2024-11-27T00:16:55,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/8ae2049e1c444fe5ba4ae2cc106993b0, entries=150, sequenceid=379, filesize=12.0 K 2024-11-27T00:16:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/c99ea9886a2d49b89776572fd42e9ba6 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c99ea9886a2d49b89776572fd42e9ba6 2024-11-27T00:16:55,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c99ea9886a2d49b89776572fd42e9ba6, entries=150, sequenceid=379, filesize=12.0 K 2024-11-27T00:16:55,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for 969e097efa67bfc5f2c39f5533a48c02 in 1273ms, sequenceid=379, compaction requested=true 2024-11-27T00:16:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:16:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:55,600 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:16:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 969e097efa67bfc5f2c39f5533a48c02:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:16:55,600 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:55,602 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:55,602 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/A is initiating minor compaction (all files) 2024-11-27T00:16:55,602 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:55,602 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/A in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:55,602 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/B is initiating minor compaction (all files) 2024-11-27T00:16:55,602 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/B in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:55,602 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/a0318dc2f0744ed2afedbc2850d72901, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d4423bcda292422b974212a4e90b58ba, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/340d00fa720a4b9ebd57721c00edcf62] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=36.8 K 2024-11-27T00:16:55,602 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/6e855e17fdf54898bdea19202ba490b4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7f1be266e176477c83b5fc32bda4013c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/8ae2049e1c444fe5ba4ae2cc106993b0] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=36.8 K 2024-11-27T00:16:55,602 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0318dc2f0744ed2afedbc2850d72901, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732666611405 2024-11-27T00:16:55,602 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e855e17fdf54898bdea19202ba490b4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732666611405 2024-11-27T00:16:55,603 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f1be266e176477c83b5fc32bda4013c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732666612141 2024-11-27T00:16:55,603 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4423bcda292422b974212a4e90b58ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732666612141 2024-11-27T00:16:55,603 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae2049e1c444fe5ba4ae2cc106993b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666613295 2024-11-27T00:16:55,603 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 340d00fa720a4b9ebd57721c00edcf62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666613295 2024-11-27T00:16:55,612 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#B#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:55,612 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:55,612 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/e29dadfd67d74114a817803786b16552 is 50, key is test_row_0/B:col10/1732666613295/Put/seqid=0 2024-11-27T00:16:55,612 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/b411c2448f92400abcd60580c6e5fb8f is 50, key is test_row_0/A:col10/1732666613295/Put/seqid=0 2024-11-27T00:16:55,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742337_1513 (size=13221) 2024-11-27T00:16:55,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742338_1514 (size=13221) 2024-11-27T00:16:56,025 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/b411c2448f92400abcd60580c6e5fb8f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/b411c2448f92400abcd60580c6e5fb8f 2024-11-27T00:16:56,027 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/e29dadfd67d74114a817803786b16552 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/e29dadfd67d74114a817803786b16552 2024-11-27T00:16:56,030 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/A of 969e097efa67bfc5f2c39f5533a48c02 into b411c2448f92400abcd60580c6e5fb8f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:56,030 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:56,030 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/A, priority=13, startTime=1732666615600; duration=0sec 2024-11-27T00:16:56,030 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:16:56,030 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:A 2024-11-27T00:16:56,031 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:16:56,031 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/B of 969e097efa67bfc5f2c39f5533a48c02 into e29dadfd67d74114a817803786b16552(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:56,031 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:56,031 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/B, priority=13, startTime=1732666615600; duration=0sec 2024-11-27T00:16:56,031 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:56,031 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:B 2024-11-27T00:16:56,031 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:16:56,031 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 969e097efa67bfc5f2c39f5533a48c02/C is initiating minor compaction (all files) 2024-11-27T00:16:56,031 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 969e097efa67bfc5f2c39f5533a48c02/C in TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:56,032 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/7b2b6b7ef1784128a092a47c330c2ce7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/353f2b3a8f9b431f8d81be7b4d6207f7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c99ea9886a2d49b89776572fd42e9ba6] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp, totalSize=36.8 K 2024-11-27T00:16:56,032 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2b6b7ef1784128a092a47c330c2ce7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732666611405 2024-11-27T00:16:56,032 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353f2b3a8f9b431f8d81be7b4d6207f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732666612141 2024-11-27T00:16:56,032 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99ea9886a2d49b89776572fd42e9ba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732666613295 2024-11-27T00:16:56,038 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 969e097efa67bfc5f2c39f5533a48c02#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:16:56,038 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/ebb96a4e7b2c439fa679558fb337106a is 50, key is test_row_0/C:col10/1732666613295/Put/seqid=0 2024-11-27T00:16:56,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742339_1515 (size=13221) 2024-11-27T00:16:56,449 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/ebb96a4e7b2c439fa679558fb337106a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/ebb96a4e7b2c439fa679558fb337106a 2024-11-27T00:16:56,455 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 969e097efa67bfc5f2c39f5533a48c02/C of 969e097efa67bfc5f2c39f5533a48c02 into ebb96a4e7b2c439fa679558fb337106a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:16:56,455 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 969e097efa67bfc5f2c39f5533a48c02: 2024-11-27T00:16:56,455 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., storeName=969e097efa67bfc5f2c39f5533a48c02/C, priority=13, startTime=1732666615600; duration=0sec 2024-11-27T00:16:56,455 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:16:56,455 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 969e097efa67bfc5f2c39f5533a48c02:C 2024-11-27T00:16:56,487 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-27T00:16:56,487 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:16:56,488 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:56,488 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:56,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:56,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1188 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3563 rows 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1184 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3552 rows 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1194 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3576 rows 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1186 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3555 rows 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1186 2024-11-27T00:16:56,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3554 rows 2024-11-27T00:16:56,490 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T00:16:56,490 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:16:56,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:56,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:56,490 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:16:56,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T00:16:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T00:16:56,497 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666616497"}]},"ts":"1732666616497"} 2024-11-27T00:16:56,499 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T00:16:56,499 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T00:16:56,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:16:56,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, UNASSIGN}] 2024-11-27T00:16:56,502 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, UNASSIGN 2024-11-27T00:16:56,503 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=969e097efa67bfc5f2c39f5533a48c02, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:56,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, UNASSIGN because future has completed 2024-11-27T00:16:56,504 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:16:56,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:16:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T00:16:56,657 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:56,657 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:16:56,658 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 969e097efa67bfc5f2c39f5533a48c02, disabling compactions & flushes 2024-11-27T00:16:56,658 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:56,658 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:56,658 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. after waiting 0 ms 2024-11-27T00:16:56,658 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:56,658 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 969e097efa67bfc5f2c39f5533a48c02 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-27T00:16:56,658 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=A 2024-11-27T00:16:56,659 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:56,659 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=B 2024-11-27T00:16:56,659 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:56,659 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 969e097efa67bfc5f2c39f5533a48c02, store=C 2024-11-27T00:16:56,659 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:16:56,665 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/baac991886d0472883b57f9a100d7206 is 50, key is test_row_1/A:col10/1732666616485/Put/seqid=0 2024-11-27T00:16:56,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742340_1516 (size=9857) 2024-11-27T00:16:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T00:16:57,071 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/baac991886d0472883b57f9a100d7206 2024-11-27T00:16:57,083 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/95ac6f194c2947d5bdbaf107cfc381ed is 50, key is test_row_1/B:col10/1732666616485/Put/seqid=0 2024-11-27T00:16:57,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742341_1517 (size=9857) 2024-11-27T00:16:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T00:16:57,489 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/95ac6f194c2947d5bdbaf107cfc381ed 2024-11-27T00:16:57,502 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/b3f7e0ef66c744c29968991e7843aece is 50, key is test_row_1/C:col10/1732666616485/Put/seqid=0 2024-11-27T00:16:57,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742342_1518 (size=9857) 2024-11-27T00:16:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T00:16:57,907 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/b3f7e0ef66c744c29968991e7843aece 2024-11-27T00:16:57,917 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/A/baac991886d0472883b57f9a100d7206 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/baac991886d0472883b57f9a100d7206 2024-11-27T00:16:57,924 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/baac991886d0472883b57f9a100d7206, entries=100, sequenceid=388, filesize=9.6 K 2024-11-27T00:16:57,925 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/B/95ac6f194c2947d5bdbaf107cfc381ed as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/95ac6f194c2947d5bdbaf107cfc381ed 2024-11-27T00:16:57,929 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/95ac6f194c2947d5bdbaf107cfc381ed, entries=100, sequenceid=388, filesize=9.6 K 2024-11-27T00:16:57,929 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/.tmp/C/b3f7e0ef66c744c29968991e7843aece as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3f7e0ef66c744c29968991e7843aece 2024-11-27T00:16:57,933 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3f7e0ef66c744c29968991e7843aece, entries=100, sequenceid=388, filesize=9.6 K 2024-11-27T00:16:57,934 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 969e097efa67bfc5f2c39f5533a48c02 in 1276ms, sequenceid=388, compaction requested=false 2024-11-27T00:16:57,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2eca29fd10914d08adfa2feabe787ce6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9871121b889648aea17244aee8edfcc4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/0b6a5125af4442e9acccd97145d84272, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ec7be938f6f04a64a8fe83bf5c3018a4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/21b3db6638f64e428c955686ed6e54bb, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/96ef72cfdfe34a0bac7f7ba38ca8af37, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/29b5e5b152c8488398f3852634d2a464, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/de9dda19fceb4e70a7dba2293149074b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/230d7c20b3f046f3b64d63a3cc0253d6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/5edbb0b780b64227b2875c48336d9637, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2124b94380f242ce9c6908f6aa62d527, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/760216f48405485981d0bb52096b42b5, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/88f1083cdf764aa78282d46f0765544a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f291e3d49e8440f285c37e10cff497f6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/e8830368c47d46988136e1720c197b4f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9959fd437120412097cec815bac4f797, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/c0257f72758b433aac1fb09c3ee3de5b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ff85ac4fc50846cea214d1e63e79e3a3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/051082cb329e4f9dbd42e95818259b55, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d5bc608b7f1e4e03a1d68e511c90ef10, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f2af2cc163ef462bb737ad4dcf670e30, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/6190fbcdd78d4238b4bcc86b151b0966, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/87a46a3f32ae420e91d596a2b7e6f00c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/22560b70f3074c16a7d338ef9365978f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/a0318dc2f0744ed2afedbc2850d72901, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d4423bcda292422b974212a4e90b58ba, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/340d00fa720a4b9ebd57721c00edcf62] to archive 2024-11-27T00:16:57,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:57,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2eca29fd10914d08adfa2feabe787ce6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2eca29fd10914d08adfa2feabe787ce6 2024-11-27T00:16:57,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9871121b889648aea17244aee8edfcc4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9871121b889648aea17244aee8edfcc4 2024-11-27T00:16:57,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/0b6a5125af4442e9acccd97145d84272 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/0b6a5125af4442e9acccd97145d84272 2024-11-27T00:16:57,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ec7be938f6f04a64a8fe83bf5c3018a4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ec7be938f6f04a64a8fe83bf5c3018a4 2024-11-27T00:16:57,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/21b3db6638f64e428c955686ed6e54bb to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/21b3db6638f64e428c955686ed6e54bb 2024-11-27T00:16:57,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/96ef72cfdfe34a0bac7f7ba38ca8af37 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/96ef72cfdfe34a0bac7f7ba38ca8af37 2024-11-27T00:16:57,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/29b5e5b152c8488398f3852634d2a464 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/29b5e5b152c8488398f3852634d2a464 2024-11-27T00:16:57,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/de9dda19fceb4e70a7dba2293149074b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/de9dda19fceb4e70a7dba2293149074b 2024-11-27T00:16:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/230d7c20b3f046f3b64d63a3cc0253d6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/230d7c20b3f046f3b64d63a3cc0253d6 2024-11-27T00:16:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/5edbb0b780b64227b2875c48336d9637 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/5edbb0b780b64227b2875c48336d9637 2024-11-27T00:16:57,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2124b94380f242ce9c6908f6aa62d527 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/2124b94380f242ce9c6908f6aa62d527 2024-11-27T00:16:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/760216f48405485981d0bb52096b42b5 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/760216f48405485981d0bb52096b42b5 2024-11-27T00:16:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/88f1083cdf764aa78282d46f0765544a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/88f1083cdf764aa78282d46f0765544a 2024-11-27T00:16:57,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f291e3d49e8440f285c37e10cff497f6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f291e3d49e8440f285c37e10cff497f6 2024-11-27T00:16:57,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/e8830368c47d46988136e1720c197b4f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/e8830368c47d46988136e1720c197b4f 2024-11-27T00:16:57,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9959fd437120412097cec815bac4f797 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/9959fd437120412097cec815bac4f797 2024-11-27T00:16:57,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/c0257f72758b433aac1fb09c3ee3de5b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/c0257f72758b433aac1fb09c3ee3de5b 2024-11-27T00:16:57,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ff85ac4fc50846cea214d1e63e79e3a3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/ff85ac4fc50846cea214d1e63e79e3a3 2024-11-27T00:16:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/051082cb329e4f9dbd42e95818259b55 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/051082cb329e4f9dbd42e95818259b55 2024-11-27T00:16:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d5bc608b7f1e4e03a1d68e511c90ef10 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d5bc608b7f1e4e03a1d68e511c90ef10 2024-11-27T00:16:57,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f2af2cc163ef462bb737ad4dcf670e30 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/f2af2cc163ef462bb737ad4dcf670e30 2024-11-27T00:16:57,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/6190fbcdd78d4238b4bcc86b151b0966 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/6190fbcdd78d4238b4bcc86b151b0966 2024-11-27T00:16:57,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/87a46a3f32ae420e91d596a2b7e6f00c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/87a46a3f32ae420e91d596a2b7e6f00c 2024-11-27T00:16:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/22560b70f3074c16a7d338ef9365978f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/22560b70f3074c16a7d338ef9365978f 2024-11-27T00:16:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/a0318dc2f0744ed2afedbc2850d72901 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/a0318dc2f0744ed2afedbc2850d72901 2024-11-27T00:16:57,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d4423bcda292422b974212a4e90b58ba to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/d4423bcda292422b974212a4e90b58ba 2024-11-27T00:16:57,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/340d00fa720a4b9ebd57721c00edcf62 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/340d00fa720a4b9ebd57721c00edcf62 2024-11-27T00:16:57,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/69728658fa924293bc9868540a7c3fee, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/88f69103eb01413f95299f58d283035f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7deec6e08dcc4813aabec7606ed0876e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b424aacbec9640f6aeb0b907280e5749, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/9c2b0d4704064506b412b31d22418f12, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/155368d6083c4a94907f817aeaeb039a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/1d8f297228b04a46bb3ae5964fcde1b1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2103c67b48fe4cdabb7cc8fe403f97b2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7d989eb6020e4d7184f4e94fbc55c9b7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2929daca9c704f7e9aebe6811d4f6f96, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/a0b50fc3d23346feba1c8b0c24a002e2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/4d8cc3ce4c2649de8f34b0ab05b55f8b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c383194c1f504db998d9c4974b0d4458, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/afb24c1b13a34312afbc6d035d656591, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/59efad090edd43af9b8f0e11e6f89302, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/93a7af8181ba4a1f96741b59bea69331, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c0e80b437aba4014a919b5adc5f9bd73, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b27f592446c6443fa32fbed8d6d70124, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/5be4710ba4fe42d49e818c3312860b26, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/aea6750850034dd5acb81a04900a3816, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c81438d54677480e9ebb2f81946384f3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/d7918b27ab1d48dbb4287ca81a6d93ea, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/659de7da36094c2a9eebc1e12e1ce77e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/6e855e17fdf54898bdea19202ba490b4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/bd4466eb0ddf45d195ac5c201196557b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7f1be266e176477c83b5fc32bda4013c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/8ae2049e1c444fe5ba4ae2cc106993b0] to archive 2024-11-27T00:16:57,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:57,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/69728658fa924293bc9868540a7c3fee to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/69728658fa924293bc9868540a7c3fee 2024-11-27T00:16:57,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/88f69103eb01413f95299f58d283035f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/88f69103eb01413f95299f58d283035f 2024-11-27T00:16:57,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7deec6e08dcc4813aabec7606ed0876e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7deec6e08dcc4813aabec7606ed0876e 2024-11-27T00:16:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b424aacbec9640f6aeb0b907280e5749 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b424aacbec9640f6aeb0b907280e5749 2024-11-27T00:16:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/9c2b0d4704064506b412b31d22418f12 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/9c2b0d4704064506b412b31d22418f12 2024-11-27T00:16:57,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/155368d6083c4a94907f817aeaeb039a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/155368d6083c4a94907f817aeaeb039a 2024-11-27T00:16:57,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/1d8f297228b04a46bb3ae5964fcde1b1 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/1d8f297228b04a46bb3ae5964fcde1b1 2024-11-27T00:16:57,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2103c67b48fe4cdabb7cc8fe403f97b2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2103c67b48fe4cdabb7cc8fe403f97b2 2024-11-27T00:16:57,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7d989eb6020e4d7184f4e94fbc55c9b7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7d989eb6020e4d7184f4e94fbc55c9b7 2024-11-27T00:16:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2929daca9c704f7e9aebe6811d4f6f96 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/2929daca9c704f7e9aebe6811d4f6f96 2024-11-27T00:16:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/a0b50fc3d23346feba1c8b0c24a002e2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/a0b50fc3d23346feba1c8b0c24a002e2 2024-11-27T00:16:57,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/4d8cc3ce4c2649de8f34b0ab05b55f8b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/4d8cc3ce4c2649de8f34b0ab05b55f8b 2024-11-27T00:16:57,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c383194c1f504db998d9c4974b0d4458 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c383194c1f504db998d9c4974b0d4458 2024-11-27T00:16:57,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/afb24c1b13a34312afbc6d035d656591 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/afb24c1b13a34312afbc6d035d656591 2024-11-27T00:16:57,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/59efad090edd43af9b8f0e11e6f89302 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/59efad090edd43af9b8f0e11e6f89302 2024-11-27T00:16:57,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/93a7af8181ba4a1f96741b59bea69331 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/93a7af8181ba4a1f96741b59bea69331 2024-11-27T00:16:57,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c0e80b437aba4014a919b5adc5f9bd73 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c0e80b437aba4014a919b5adc5f9bd73 2024-11-27T00:16:57,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b27f592446c6443fa32fbed8d6d70124 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/b27f592446c6443fa32fbed8d6d70124 2024-11-27T00:16:57,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/5be4710ba4fe42d49e818c3312860b26 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/5be4710ba4fe42d49e818c3312860b26 2024-11-27T00:16:57,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/aea6750850034dd5acb81a04900a3816 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/aea6750850034dd5acb81a04900a3816 2024-11-27T00:16:57,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c81438d54677480e9ebb2f81946384f3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/c81438d54677480e9ebb2f81946384f3 2024-11-27T00:16:57,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/d7918b27ab1d48dbb4287ca81a6d93ea to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/d7918b27ab1d48dbb4287ca81a6d93ea 2024-11-27T00:16:57,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/659de7da36094c2a9eebc1e12e1ce77e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/659de7da36094c2a9eebc1e12e1ce77e 2024-11-27T00:16:57,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/6e855e17fdf54898bdea19202ba490b4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/6e855e17fdf54898bdea19202ba490b4 2024-11-27T00:16:57,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/bd4466eb0ddf45d195ac5c201196557b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/bd4466eb0ddf45d195ac5c201196557b 2024-11-27T00:16:57,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7f1be266e176477c83b5fc32bda4013c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/7f1be266e176477c83b5fc32bda4013c 2024-11-27T00:16:57,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/8ae2049e1c444fe5ba4ae2cc106993b0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/8ae2049e1c444fe5ba4ae2cc106993b0 2024-11-27T00:16:57,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/869fd4cad8014bb2af6be8d4b4b78797, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/efa1f1f3f6a94c438f54b998631b4af9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3d2f499d7bcf4b6f86aea2ca8e5e7b9a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/4b34d351457d4da59d102a55946013a0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ad3d4eee6a649b79c479b921e0df8ea, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/146a627c8b6c479db0386095471319f4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/44ff29db89964309aa5f4d9d726e275f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0b044c5069394f75bc38dd5f3fe28718, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5773bbf25f9d4e14a67acb885875a3e8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ee72df2b3ac414b8b60bd864dc3b0f6, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c13eb00434a349ae99d89879cbc88b85, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/668a2a7f393f4bc898ebbcb7f953a078, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/686dd073304d4ea09da2f8da02612be8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/52d4a889ab2f45f8a504ff004116e6e2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/804f07685ff34a60afb40a44615620e4, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3e9d099533f84d00835b2f26e1ea636a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/676842bab2864077943f1a3a959fe98b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/936df0d627e34cebbb2401db9ef241f9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/93c7a31155964e4bbb1b12fcf20f0213, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/27ab760488f749e2bf49e502131f0a87, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/197e350bc3a041bcb049e8ebd816fd78, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5a904225322b4b679835d3b86605761b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/e60111b16b3344768872f9c4e2bd0351, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/7b2b6b7ef1784128a092a47c330c2ce7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3ceda0138cc4afb802c0d89a57362cc, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/353f2b3a8f9b431f8d81be7b4d6207f7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c99ea9886a2d49b89776572fd42e9ba6] to archive 2024-11-27T00:16:57,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:16:57,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/869fd4cad8014bb2af6be8d4b4b78797 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/869fd4cad8014bb2af6be8d4b4b78797 2024-11-27T00:16:57,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/efa1f1f3f6a94c438f54b998631b4af9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/efa1f1f3f6a94c438f54b998631b4af9 2024-11-27T00:16:57,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3d2f499d7bcf4b6f86aea2ca8e5e7b9a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3d2f499d7bcf4b6f86aea2ca8e5e7b9a 2024-11-27T00:16:57,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/4b34d351457d4da59d102a55946013a0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/4b34d351457d4da59d102a55946013a0 2024-11-27T00:16:57,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ad3d4eee6a649b79c479b921e0df8ea to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ad3d4eee6a649b79c479b921e0df8ea 2024-11-27T00:16:57,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/146a627c8b6c479db0386095471319f4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/146a627c8b6c479db0386095471319f4 2024-11-27T00:16:57,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/44ff29db89964309aa5f4d9d726e275f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/44ff29db89964309aa5f4d9d726e275f 2024-11-27T00:16:57,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0b044c5069394f75bc38dd5f3fe28718 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0b044c5069394f75bc38dd5f3fe28718 2024-11-27T00:16:57,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5773bbf25f9d4e14a67acb885875a3e8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5773bbf25f9d4e14a67acb885875a3e8 2024-11-27T00:16:57,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ee72df2b3ac414b8b60bd864dc3b0f6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/0ee72df2b3ac414b8b60bd864dc3b0f6 2024-11-27T00:16:57,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c13eb00434a349ae99d89879cbc88b85 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c13eb00434a349ae99d89879cbc88b85 2024-11-27T00:16:57,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/668a2a7f393f4bc898ebbcb7f953a078 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/668a2a7f393f4bc898ebbcb7f953a078 2024-11-27T00:16:57,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/686dd073304d4ea09da2f8da02612be8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/686dd073304d4ea09da2f8da02612be8 2024-11-27T00:16:57,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/52d4a889ab2f45f8a504ff004116e6e2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/52d4a889ab2f45f8a504ff004116e6e2 2024-11-27T00:16:57,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/804f07685ff34a60afb40a44615620e4 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/804f07685ff34a60afb40a44615620e4 2024-11-27T00:16:57,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3e9d099533f84d00835b2f26e1ea636a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/3e9d099533f84d00835b2f26e1ea636a 2024-11-27T00:16:58,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/676842bab2864077943f1a3a959fe98b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/676842bab2864077943f1a3a959fe98b 2024-11-27T00:16:58,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/936df0d627e34cebbb2401db9ef241f9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/936df0d627e34cebbb2401db9ef241f9 2024-11-27T00:16:58,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/93c7a31155964e4bbb1b12fcf20f0213 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/93c7a31155964e4bbb1b12fcf20f0213 2024-11-27T00:16:58,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/27ab760488f749e2bf49e502131f0a87 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/27ab760488f749e2bf49e502131f0a87 2024-11-27T00:16:58,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/197e350bc3a041bcb049e8ebd816fd78 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/197e350bc3a041bcb049e8ebd816fd78 2024-11-27T00:16:58,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5a904225322b4b679835d3b86605761b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/5a904225322b4b679835d3b86605761b 2024-11-27T00:16:58,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/e60111b16b3344768872f9c4e2bd0351 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/e60111b16b3344768872f9c4e2bd0351 2024-11-27T00:16:58,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/7b2b6b7ef1784128a092a47c330c2ce7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/7b2b6b7ef1784128a092a47c330c2ce7 2024-11-27T00:16:58,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3ceda0138cc4afb802c0d89a57362cc to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3ceda0138cc4afb802c0d89a57362cc 2024-11-27T00:16:58,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/353f2b3a8f9b431f8d81be7b4d6207f7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/353f2b3a8f9b431f8d81be7b4d6207f7 2024-11-27T00:16:58,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c99ea9886a2d49b89776572fd42e9ba6 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/c99ea9886a2d49b89776572fd42e9ba6 2024-11-27T00:16:58,010 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/recovered.edits/391.seqid, newMaxSeqId=391, maxSeqId=1 2024-11-27T00:16:58,010 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. 2024-11-27T00:16:58,010 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 969e097efa67bfc5f2c39f5533a48c02: Waiting for close lock at 1732666616657Running coprocessor pre-close hooks at 1732666616657Disabling compacts and flushes for region at 1732666616657Disabling writes for close at 1732666616658 (+1 ms)Obtaining lock to block concurrent updates at 1732666616658Preparing flush snapshotting stores in 969e097efa67bfc5f2c39f5533a48c02 at 1732666616658Finished memstore snapshotting TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732666616659 (+1 ms)Flushing stores of TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02. at 1732666616660 (+1 ms)Flushing 969e097efa67bfc5f2c39f5533a48c02/A: creating writer at 1732666616660Flushing 969e097efa67bfc5f2c39f5533a48c02/A: appending metadata at 1732666616665 (+5 ms)Flushing 969e097efa67bfc5f2c39f5533a48c02/A: closing flushed file at 1732666616665Flushing 969e097efa67bfc5f2c39f5533a48c02/B: creating writer at 1732666617079 (+414 ms)Flushing 969e097efa67bfc5f2c39f5533a48c02/B: appending metadata at 1732666617082 (+3 ms)Flushing 969e097efa67bfc5f2c39f5533a48c02/B: closing flushed file at 1732666617082Flushing 969e097efa67bfc5f2c39f5533a48c02/C: creating writer at 1732666617497 (+415 ms)Flushing 969e097efa67bfc5f2c39f5533a48c02/C: appending metadata at 1732666617501 (+4 ms)Flushing 969e097efa67bfc5f2c39f5533a48c02/C: closing flushed file at 1732666617501Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f12db52: reopening flushed file at 1732666617916 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63717e33: reopening flushed file at 1732666617924 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74858694: reopening flushed file at 1732666617929 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 969e097efa67bfc5f2c39f5533a48c02 in 1276ms, sequenceid=388, compaction requested=false at 1732666617934 (+5 ms)Writing region close event to WAL at 1732666618008 (+74 ms)Running coprocessor post-close hooks at 1732666618010 (+2 ms)Closed at 1732666618010 2024-11-27T00:16:58,012 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:58,012 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=969e097efa67bfc5f2c39f5533a48c02, regionState=CLOSED 2024-11-27T00:16:58,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:16:58,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-27T00:16:58,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 969e097efa67bfc5f2c39f5533a48c02, server=f036bc4986ea,33881,1732666453890 in 1.5100 sec 2024-11-27T00:16:58,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-27T00:16:58,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=969e097efa67bfc5f2c39f5533a48c02, UNASSIGN in 1.5140 sec 2024-11-27T00:16:58,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-27T00:16:58,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5170 sec 2024-11-27T00:16:58,019 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666618018"}]},"ts":"1732666618018"} 2024-11-27T00:16:58,020 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T00:16:58,020 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T00:16:58,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5280 sec 2024-11-27T00:16:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T00:16:58,644 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:58,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T00:16:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:58,647 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T00:16:58,647 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:58,650 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:58,652 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/recovered.edits] 2024-11-27T00:16:58,654 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/b411c2448f92400abcd60580c6e5fb8f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/b411c2448f92400abcd60580c6e5fb8f 2024-11-27T00:16:58,655 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/baac991886d0472883b57f9a100d7206 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/A/baac991886d0472883b57f9a100d7206 2024-11-27T00:16:58,656 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/95ac6f194c2947d5bdbaf107cfc381ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/95ac6f194c2947d5bdbaf107cfc381ed 2024-11-27T00:16:58,657 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/e29dadfd67d74114a817803786b16552 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/B/e29dadfd67d74114a817803786b16552 2024-11-27T00:16:58,659 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3f7e0ef66c744c29968991e7843aece to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/b3f7e0ef66c744c29968991e7843aece 2024-11-27T00:16:58,660 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/ebb96a4e7b2c439fa679558fb337106a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/C/ebb96a4e7b2c439fa679558fb337106a 2024-11-27T00:16:58,662 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/recovered.edits/391.seqid to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02/recovered.edits/391.seqid 2024-11-27T00:16:58,662 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/969e097efa67bfc5f2c39f5533a48c02 2024-11-27T00:16:58,662 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T00:16:58,664 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:58,666 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T00:16:58,667 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T00:16:58,668 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:58,668 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T00:16:58,669 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732666618668"}]},"ts":"9223372036854775807"} 2024-11-27T00:16:58,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T00:16:58,670 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 969e097efa67bfc5f2c39f5533a48c02, NAME => 'TestAcidGuarantees,,1732666591126.969e097efa67bfc5f2c39f5533a48c02.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T00:16:58,670 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T00:16:58,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732666618671"}]},"ts":"9223372036854775807"} 2024-11-27T00:16:58,672 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T00:16:58,672 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:58,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-27T00:16:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T00:16:58,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T00:16:58,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:58,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 249) - Thread LEAK? -, OpenFileDescriptor=457 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=401 (was 385) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2224 (was 2229) 2024-11-27T00:16:58,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=401, ProcessCount=11, AvailableMemoryMB=2224 2024-11-27T00:16:58,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:16:58,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:16:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:58,787 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T00:16:58,787 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:58,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-27T00:16:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T00:16:58,788 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T00:16:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742343_1519 (size=1061) 2024-11-27T00:16:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T00:16:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T00:16:59,199 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772 2024-11-27T00:16:59,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742344_1520 (size=53) 2024-11-27T00:16:59,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:16:59,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 199009045896d5ba974e92cd43d397ed, disabling compactions & flushes 2024-11-27T00:16:59,209 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:16:59,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:16:59,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. after waiting 0 ms 2024-11-27T00:16:59,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:16:59,209 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:16:59,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 199009045896d5ba974e92cd43d397ed: Waiting for close lock at 1732666619209Disabling compacts and flushes for region at 1732666619209Disabling writes for close at 1732666619209Writing region close event to WAL at 1732666619209Closed at 1732666619209 2024-11-27T00:16:59,211 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T00:16:59,211 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732666619211"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732666619211"}]},"ts":"1732666619211"} 2024-11-27T00:16:59,213 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T00:16:59,214 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T00:16:59,215 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666619214"}]},"ts":"1732666619214"} 2024-11-27T00:16:59,217 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T00:16:59,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, ASSIGN}] 2024-11-27T00:16:59,219 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, ASSIGN 2024-11-27T00:16:59,220 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, ASSIGN; state=OFFLINE, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=false 2024-11-27T00:16:59,371 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=199009045896d5ba974e92cd43d397ed, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:59,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, ASSIGN because future has completed 2024-11-27T00:16:59,376 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:16:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T00:16:59,532 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:16:59,532 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:16:59,532 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,532 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:16:59,532 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,532 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,533 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,534 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:16:59,534 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 199009045896d5ba974e92cd43d397ed columnFamilyName A 2024-11-27T00:16:59,535 DEBUG [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:59,535 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(327): Store=199009045896d5ba974e92cd43d397ed/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:16:59,535 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,536 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:16:59,536 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 199009045896d5ba974e92cd43d397ed columnFamilyName B 2024-11-27T00:16:59,536 DEBUG [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:59,536 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(327): Store=199009045896d5ba974e92cd43d397ed/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:16:59,537 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,537 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:16:59,537 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 199009045896d5ba974e92cd43d397ed columnFamilyName C 2024-11-27T00:16:59,538 DEBUG [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:16:59,538 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(327): Store=199009045896d5ba974e92cd43d397ed/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:16:59,538 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:16:59,538 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,539 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,539 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,540 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,540 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,540 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:16:59,541 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,542 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T00:16:59,542 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 199009045896d5ba974e92cd43d397ed; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65254350, jitterRate=-0.027634412050247192}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:16:59,542 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:16:59,543 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 199009045896d5ba974e92cd43d397ed: Running coprocessor pre-open hook at 1732666619532Writing region info on filesystem at 1732666619532Initializing all the Stores at 1732666619533 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666619533Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666619533Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666619533Cleaning up temporary data from old regions at 1732666619540 (+7 ms)Running coprocessor post-open hooks at 1732666619542 (+2 ms)Region opened successfully at 1732666619543 (+1 ms) 2024-11-27T00:16:59,543 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., pid=148, masterSystemTime=1732666619529 2024-11-27T00:16:59,545 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:16:59,545 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:16:59,545 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=199009045896d5ba974e92cd43d397ed, regionState=OPEN, openSeqNum=2, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:16:59,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:16:59,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-27T00:16:59,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 in 171 msec 2024-11-27T00:16:59,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-27T00:16:59,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, ASSIGN in 332 msec 2024-11-27T00:16:59,551 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T00:16:59,551 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666619551"}]},"ts":"1732666619551"} 2024-11-27T00:16:59,552 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T00:16:59,552 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T00:16:59,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 767 msec 2024-11-27T00:16:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T00:16:59,925 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:16:59,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:59,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:16:59,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:16:59,929 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:16:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:16:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:16:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@961586e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:16:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:16:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:16:59,932 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:16:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58684c67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:16:59,934 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:16:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:16:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:16:59,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:16:59,938 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T00:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is f036bc4986ea,33599,1732666453115 2024-11-27T00:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@72dc655 2024-11-27T00:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T00:16:59,940 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T00:16:59,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T00:16:59,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T00:16:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T00:16:59,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742345_1521 (size=1097) 2024-11-27T00:17:00,361 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-27T00:17:00,361 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-27T00:17:00,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:17:00,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, REOPEN/MOVE}] 2024-11-27T00:17:00,370 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, REOPEN/MOVE 2024-11-27T00:17:00,370 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=199009045896d5ba974e92cd43d397ed, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:00,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, REOPEN/MOVE because future has completed 2024-11-27T00:17:00,372 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:17:00,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:17:00,526 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,526 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:17:00,526 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 199009045896d5ba974e92cd43d397ed, disabling compactions & flushes 2024-11-27T00:17:00,526 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:00,526 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:00,526 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. after waiting 0 ms 2024-11-27T00:17:00,527 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:00,534 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T00:17:00,535 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:00,535 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 199009045896d5ba974e92cd43d397ed: Waiting for close lock at 1732666620526Running coprocessor pre-close hooks at 1732666620526Disabling compacts and flushes for region at 1732666620526Disabling writes for close at 1732666620526Writing region close event to WAL at 1732666620528 (+2 ms)Running coprocessor post-close hooks at 1732666620535 (+7 ms)Closed at 1732666620535 2024-11-27T00:17:00,535 WARN [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 199009045896d5ba974e92cd43d397ed to self. 2024-11-27T00:17:00,537 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,538 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=199009045896d5ba974e92cd43d397ed, regionState=CLOSED 2024-11-27T00:17:00,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:17:00,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-27T00:17:00,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 in 169 msec 2024-11-27T00:17:00,544 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, REOPEN/MOVE; state=CLOSED, location=f036bc4986ea,33881,1732666453890; forceNewPlan=false, retain=true 2024-11-27T00:17:00,695 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=199009045896d5ba974e92cd43d397ed, regionState=OPENING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:00,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, REOPEN/MOVE because future has completed 2024-11-27T00:17:00,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:17:00,856 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:00,856 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} 2024-11-27T00:17:00,856 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,856 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T00:17:00,856 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,857 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,858 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,858 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:17:00,858 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 199009045896d5ba974e92cd43d397ed columnFamilyName A 2024-11-27T00:17:00,859 DEBUG [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:00,859 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(327): Store=199009045896d5ba974e92cd43d397ed/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:17:00,860 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,860 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:17:00,860 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 199009045896d5ba974e92cd43d397ed columnFamilyName B 2024-11-27T00:17:00,860 DEBUG [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:00,860 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(327): Store=199009045896d5ba974e92cd43d397ed/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:17:00,860 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,861 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-27T00:17:00,861 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 199009045896d5ba974e92cd43d397ed columnFamilyName C 2024-11-27T00:17:00,861 DEBUG [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:00,861 INFO [StoreOpener-199009045896d5ba974e92cd43d397ed-1 {}] regionserver.HStore(327): Store=199009045896d5ba974e92cd43d397ed/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T00:17:00,861 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:00,861 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,862 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,863 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,863 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,864 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,864 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T00:17:00,865 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,865 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 199009045896d5ba974e92cd43d397ed; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70093689, jitterRate=0.04447735846042633}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T00:17:00,865 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,866 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 199009045896d5ba974e92cd43d397ed: Running coprocessor pre-open hook at 1732666620857Writing region info on filesystem at 1732666620857Initializing all the Stores at 1732666620857Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666620857Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666620857Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732666620857Cleaning up temporary data from old regions at 1732666620864 (+7 ms)Running coprocessor post-open hooks at 1732666620865 (+1 ms)Region opened successfully at 1732666620866 (+1 ms) 2024-11-27T00:17:00,866 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., pid=153, masterSystemTime=1732666620853 2024-11-27T00:17:00,868 DEBUG [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:00,868 INFO [RS_OPEN_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:00,868 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=199009045896d5ba974e92cd43d397ed, regionState=OPEN, openSeqNum=5, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:00,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:17:00,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-27T00:17:00,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 in 170 msec 2024-11-27T00:17:00,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-27T00:17:00,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, REOPEN/MOVE in 503 msec 2024-11-27T00:17:00,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-27T00:17:00,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-27T00:17:00,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 933 msec 2024-11-27T00:17:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-27T00:17:00,878 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:00,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,879 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed718cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63603f77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,882 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f407d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,883 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d56dd4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,884 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56dbacb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,886 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d176eb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,887 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13f09d74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,888 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18fc758d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,889 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481b1e83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,890 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677db894, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,892 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37aee339, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33556c9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,895 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229b59c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d01473c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,897 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd25cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,898 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@557fd901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,900 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b007fdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,901 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a1ef77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f036bc4986ea,33599,-1 for getting cluster id 2024-11-27T00:17:00,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T00:17:00,902 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0' 2024-11-27T00:17:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T00:17:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8fc44ded-9cd1-4a60-aa36-1f9cdcacfad0" 2024-11-27T00:17:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24829ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f036bc4986ea,33599,-1] 2024-11-27T00:17:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T00:17:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:00,903 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T00:17:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@449e92f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T00:17:00,904 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,904 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,904 DEBUG [Thread-2337 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,905 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,905 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,905 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,905 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-27T00:17:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,906 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,906 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,907 DEBUG [Thread-2341 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,907 DEBUG [Thread-2341 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,908 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,908 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,909 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T00:17:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f036bc4986ea,33881,1732666453890, seqNum=-1] 2024-11-27T00:17:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T00:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T00:17:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:00,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T00:17:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5] 2024-11-27T00:17:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:00,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:17:00,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:00,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:00,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:00,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:00,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:00,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666680930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666620929, max timestamp=1732666620934Num uniques 859; 2024-11-27T00:17:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666620929, max timestamp=1732666620934Num uniques 859; 2024-11-27T00:17:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666620929, max timestamp=1732666620934Num uniques 859; 2024-11-27T00:17:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:00,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273baa2dcc3c394629b6f751db841d3650_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666620915/Put/seqid=0 2024-11-27T00:17:00,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742346_1522 (size=14594) 2024-11-27T00:17:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666680954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666680954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:00,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666680959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:00,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666680961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T00:17:01,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666681044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:01,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:01,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666681063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666681064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666681074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666681074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:01,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:01,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T00:17:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666681253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666681274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666681274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666681284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666681284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,351 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:01,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273baa2dcc3c394629b6f751db841d3650_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273baa2dcc3c394629b6f751db841d3650_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:01,354 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/ae3bc999367949baa89b99791a7dee17, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:01,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/ae3bc999367949baa89b99791a7dee17 is 175, key is test_row_0/A:col10/1732666620915/Put/seqid=0 2024-11-27T00:17:01,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742347_1523 (size=39545) 2024-11-27T00:17:01,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:01,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T00:17:01,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666681563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666681584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666681584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666681594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666681594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:01,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:01,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:01,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,758 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/ae3bc999367949baa89b99791a7dee17 2024-11-27T00:17:01,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/6ce336cfd92c446682e09843c9cce7ea is 50, key is test_row_0/B:col10/1732666620915/Put/seqid=0 2024-11-27T00:17:01,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742348_1524 (size=9657) 2024-11-27T00:17:01,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:01,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:01,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:01,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T00:17:02,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666682074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666682094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666682094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:02,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666682103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:02,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666682103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:02,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:02,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:02,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:02,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:02,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:02,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:02,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/6ce336cfd92c446682e09843c9cce7ea 2024-11-27T00:17:02,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/1ef765f3ac7841438fae5fc852b70b26 is 50, key is test_row_0/C:col10/1732666620915/Put/seqid=0 2024-11-27T00:17:02,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742349_1525 (size=9657) 2024-11-27T00:17:02,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/1ef765f3ac7841438fae5fc852b70b26 2024-11-27T00:17:02,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/ae3bc999367949baa89b99791a7dee17 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ae3bc999367949baa89b99791a7dee17 2024-11-27T00:17:02,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ae3bc999367949baa89b99791a7dee17, entries=200, sequenceid=15, filesize=38.6 K 2024-11-27T00:17:02,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/6ce336cfd92c446682e09843c9cce7ea as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6ce336cfd92c446682e09843c9cce7ea 2024-11-27T00:17:02,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6ce336cfd92c446682e09843c9cce7ea, entries=100, sequenceid=15, filesize=9.4 K 2024-11-27T00:17:02,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/1ef765f3ac7841438fae5fc852b70b26 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1ef765f3ac7841438fae5fc852b70b26 2024-11-27T00:17:02,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1ef765f3ac7841438fae5fc852b70b26, entries=100, sequenceid=15, filesize=9.4 K 2024-11-27T00:17:02,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 199009045896d5ba974e92cd43d397ed in 1312ms, sequenceid=15, compaction requested=false 2024-11-27T00:17:02,229 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T00:17:02,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:02,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T00:17:02,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:02,285 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-27T00:17:02,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:02,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:02,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:02,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:02,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:02,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275706c5461da14e5085849102aebaab9b_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666620957/Put/seqid=0 2024-11-27T00:17:02,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742350_1526 (size=12154) 2024-11-27T00:17:02,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:02,706 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275706c5461da14e5085849102aebaab9b_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275706c5461da14e5085849102aebaab9b_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:02,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/4e697e2fb8b042db8887bce19768dd81, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:02,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/4e697e2fb8b042db8887bce19768dd81 is 175, key is test_row_0/A:col10/1732666620957/Put/seqid=0 2024-11-27T00:17:02,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742351_1527 (size=30955) 2024-11-27T00:17:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T00:17:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:03,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:03,110 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/4e697e2fb8b042db8887bce19768dd81 2024-11-27T00:17:03,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666683107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666683108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666683110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/0f5d57ebfe8a47fdb37d4e4469f583f8 is 50, key is test_row_0/B:col10/1732666620957/Put/seqid=0 2024-11-27T00:17:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666683114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,118 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742352_1528 (size=12001) 2024-11-27T00:17:03,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666683123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,127 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666683223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666683223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666683223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666683434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666683434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666683434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,520 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/0f5d57ebfe8a47fdb37d4e4469f583f8 2024-11-27T00:17:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/6298175c826c4a82a6327517e0a0426b is 50, key is test_row_0/C:col10/1732666620957/Put/seqid=0 2024-11-27T00:17:03,527 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T00:17:03,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742353_1529 (size=12001) 2024-11-27T00:17:03,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666683744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666683744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666683744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:03,936 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/6298175c826c4a82a6327517e0a0426b 2024-11-27T00:17:03,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/4e697e2fb8b042db8887bce19768dd81 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e697e2fb8b042db8887bce19768dd81 2024-11-27T00:17:03,943 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e697e2fb8b042db8887bce19768dd81, entries=150, sequenceid=50, filesize=30.2 K 2024-11-27T00:17:03,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/0f5d57ebfe8a47fdb37d4e4469f583f8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/0f5d57ebfe8a47fdb37d4e4469f583f8 2024-11-27T00:17:03,947 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/0f5d57ebfe8a47fdb37d4e4469f583f8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T00:17:03,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/6298175c826c4a82a6327517e0a0426b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/6298175c826c4a82a6327517e0a0426b 2024-11-27T00:17:03,950 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/6298175c826c4a82a6327517e0a0426b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T00:17:03,951 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 199009045896d5ba974e92cd43d397ed in 1666ms, sequenceid=50, compaction requested=false 2024-11-27T00:17:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-27T00:17:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-27T00:17:03,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-27T00:17:03,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0410 sec 2024-11-27T00:17:03,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.0490 sec 2024-11-27T00:17:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:04,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-27T00:17:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:04,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270e803ab932ee4bfb80a4bb3bb201b530_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666624255/Put/seqid=0 2024-11-27T00:17:04,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742354_1530 (size=14594) 2024-11-27T00:17:04,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:04,279 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270e803ab932ee4bfb80a4bb3bb201b530_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270e803ab932ee4bfb80a4bb3bb201b530_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:04,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/0e8cb4b523c341678059cc4a6f7b0f4a, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:04,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/0e8cb4b523c341678059cc4a6f7b0f4a is 175, key is test_row_0/A:col10/1732666624255/Put/seqid=0 2024-11-27T00:17:04,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742355_1531 (size=39549) 2024-11-27T00:17:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666684282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666684284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666684285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666684394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666684394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666684394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666684604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666684604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666684604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/0e8cb4b523c341678059cc4a6f7b0f4a 2024-11-27T00:17:04,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/1e39e95f89a94a9ab085e92fa658c01e is 50, key is test_row_0/B:col10/1732666624255/Put/seqid=0 2024-11-27T00:17:04,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742356_1532 (size=12001) 2024-11-27T00:17:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666684914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666684914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666684914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:04,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T00:17:05,064 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:05,064 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-27T00:17:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T00:17:05,065 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:05,066 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:05,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:05,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/1e39e95f89a94a9ab085e92fa658c01e 2024-11-27T00:17:05,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/72b04f9ed3334ec8b89ce3487f9393e8 is 50, key is test_row_0/C:col10/1732666624255/Put/seqid=0 2024-11-27T00:17:05,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742357_1533 (size=12001) 2024-11-27T00:17:05,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666685143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:05,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666685143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:05,146 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:05,146 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T00:17:05,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T00:17:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T00:17:05,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:05,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T00:17:05,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666685424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:05,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666685424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:05,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666685424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:05,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/72b04f9ed3334ec8b89ce3487f9393e8 2024-11-27T00:17:05,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/0e8cb4b523c341678059cc4a6f7b0f4a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/0e8cb4b523c341678059cc4a6f7b0f4a 2024-11-27T00:17:05,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/0e8cb4b523c341678059cc4a6f7b0f4a, entries=200, sequenceid=64, filesize=38.6 K 2024-11-27T00:17:05,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/1e39e95f89a94a9ab085e92fa658c01e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1e39e95f89a94a9ab085e92fa658c01e 2024-11-27T00:17:05,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1e39e95f89a94a9ab085e92fa658c01e, entries=150, sequenceid=64, filesize=11.7 K 2024-11-27T00:17:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/72b04f9ed3334ec8b89ce3487f9393e8 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/72b04f9ed3334ec8b89ce3487f9393e8 2024-11-27T00:17:05,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T00:17:05,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:05,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:05,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/72b04f9ed3334ec8b89ce3487f9393e8, entries=150, sequenceid=64, filesize=11.7 K 2024-11-27T00:17:05,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 199009045896d5ba974e92cd43d397ed in 1266ms, sequenceid=64, compaction requested=true 2024-11-27T00:17:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:05,524 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:17:05,524 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:05,525 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:05,525 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/B is initiating minor compaction (all files) 2024-11-27T00:17:05,525 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/B in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,525 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6ce336cfd92c446682e09843c9cce7ea, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/0f5d57ebfe8a47fdb37d4e4469f583f8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1e39e95f89a94a9ab085e92fa658c01e] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=32.9 K 2024-11-27T00:17:05,525 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce336cfd92c446682e09843c9cce7ea, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732666620915 2024-11-27T00:17:05,525 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:05,525 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/A is initiating minor compaction (all files) 2024-11-27T00:17:05,525 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/A in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,525 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f5d57ebfe8a47fdb37d4e4469f583f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732666620949 2024-11-27T00:17:05,525 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ae3bc999367949baa89b99791a7dee17, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e697e2fb8b042db8887bce19768dd81, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/0e8cb4b523c341678059cc4a6f7b0f4a] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=107.5 K 2024-11-27T00:17:05,525 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,525 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ae3bc999367949baa89b99791a7dee17, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e697e2fb8b042db8887bce19768dd81, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/0e8cb4b523c341678059cc4a6f7b0f4a] 2024-11-27T00:17:05,526 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e39e95f89a94a9ab085e92fa658c01e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732666624255 2024-11-27T00:17:05,526 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3bc999367949baa89b99791a7dee17, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732666620913 2024-11-27T00:17:05,526 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e697e2fb8b042db8887bce19768dd81, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732666620949 2024-11-27T00:17:05,526 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e8cb4b523c341678059cc4a6f7b0f4a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732666623108 2024-11-27T00:17:05,530 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:05,531 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:05,531 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/05bab48d8868478281792e31a6a41f83 is 50, key is test_row_0/B:col10/1732666624255/Put/seqid=0 2024-11-27T00:17:05,532 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b740cd7308cc484180d7068cf0fa1cb6_199009045896d5ba974e92cd43d397ed store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:05,534 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b740cd7308cc484180d7068cf0fa1cb6_199009045896d5ba974e92cd43d397ed, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:05,534 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b740cd7308cc484180d7068cf0fa1cb6_199009045896d5ba974e92cd43d397ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:05,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742358_1534 (size=12104) 2024-11-27T00:17:05,542 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/05bab48d8868478281792e31a6a41f83 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/05bab48d8868478281792e31a6a41f83 2024-11-27T00:17:05,545 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/B of 199009045896d5ba974e92cd43d397ed into 05bab48d8868478281792e31a6a41f83(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:05,545 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:05,545 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/B, priority=13, startTime=1732666625524; duration=0sec 2024-11-27T00:17:05,546 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:05,546 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:05,546 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:05,546 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:05,546 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/C is initiating minor compaction (all files) 2024-11-27T00:17:05,546 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/C in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,547 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1ef765f3ac7841438fae5fc852b70b26, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/6298175c826c4a82a6327517e0a0426b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/72b04f9ed3334ec8b89ce3487f9393e8] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=32.9 K 2024-11-27T00:17:05,547 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef765f3ac7841438fae5fc852b70b26, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732666620915 2024-11-27T00:17:05,547 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6298175c826c4a82a6327517e0a0426b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732666620949 2024-11-27T00:17:05,547 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b04f9ed3334ec8b89ce3487f9393e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732666624255 2024-11-27T00:17:05,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742359_1535 (size=4469) 2024-11-27T00:17:05,559 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#C#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:05,559 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/81ba8c22f61547a39e1e0234bcd5d825 is 50, key is test_row_0/C:col10/1732666624255/Put/seqid=0 2024-11-27T00:17:05,562 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#A#compaction#459 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:05,562 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/ddf73f6c26f04bbc8d438ff652c834b2 is 175, key is test_row_0/A:col10/1732666624255/Put/seqid=0 2024-11-27T00:17:05,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742360_1536 (size=12104) 2024-11-27T00:17:05,584 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/81ba8c22f61547a39e1e0234bcd5d825 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/81ba8c22f61547a39e1e0234bcd5d825 2024-11-27T00:17:05,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742361_1537 (size=31058) 2024-11-27T00:17:05,588 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/C of 199009045896d5ba974e92cd43d397ed into 81ba8c22f61547a39e1e0234bcd5d825(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:05,588 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:05,588 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/C, priority=13, startTime=1732666625524; duration=0sec 2024-11-27T00:17:05,589 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:05,589 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:05,591 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/ddf73f6c26f04bbc8d438ff652c834b2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ddf73f6c26f04bbc8d438ff652c834b2 2024-11-27T00:17:05,595 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/A of 199009045896d5ba974e92cd43d397ed into ddf73f6c26f04bbc8d438ff652c834b2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:05,595 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:05,595 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/A, priority=13, startTime=1732666625524; duration=0sec 2024-11-27T00:17:05,595 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:05,595 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:05,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T00:17:05,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:05,675 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-27T00:17:05,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:05,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:05,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:05,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:05,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:05,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:05,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127aeac82613160432aba9396fa1fc43350_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666624284/Put/seqid=0 2024-11-27T00:17:05,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742362_1538 (size=12154) 2024-11-27T00:17:05,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:05,691 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127aeac82613160432aba9396fa1fc43350_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aeac82613160432aba9396fa1fc43350_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:05,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/5fa3b9b9d50949209ecebe0a3181d8d3, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:05,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/5fa3b9b9d50949209ecebe0a3181d8d3 is 175, key is test_row_0/A:col10/1732666624284/Put/seqid=0 2024-11-27T00:17:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T00:17:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742363_1539 (size=30955) 2024-11-27T00:17:05,698 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/5fa3b9b9d50949209ecebe0a3181d8d3 2024-11-27T00:17:05,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/5403256cdc3c490b83004339b93d2e94 is 50, key is test_row_0/B:col10/1732666624284/Put/seqid=0 2024-11-27T00:17:05,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742364_1540 (size=12001) 2024-11-27T00:17:06,109 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/5403256cdc3c490b83004339b93d2e94 2024-11-27T00:17:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/1a4ec1f11c5f48b18ee1169574549629 is 50, key is test_row_0/C:col10/1732666624284/Put/seqid=0 2024-11-27T00:17:06,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742365_1541 (size=12001) 2024-11-27T00:17:06,130 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/1a4ec1f11c5f48b18ee1169574549629 2024-11-27T00:17:06,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/5fa3b9b9d50949209ecebe0a3181d8d3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5fa3b9b9d50949209ecebe0a3181d8d3 2024-11-27T00:17:06,137 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5fa3b9b9d50949209ecebe0a3181d8d3, entries=150, sequenceid=85, filesize=30.2 K 2024-11-27T00:17:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/5403256cdc3c490b83004339b93d2e94 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5403256cdc3c490b83004339b93d2e94 2024-11-27T00:17:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,141 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5403256cdc3c490b83004339b93d2e94, entries=150, sequenceid=85, filesize=11.7 K 2024-11-27T00:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/1a4ec1f11c5f48b18ee1169574549629 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1a4ec1f11c5f48b18ee1169574549629 2024-11-27T00:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,145 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1a4ec1f11c5f48b18ee1169574549629, entries=150, sequenceid=85, filesize=11.7 K 2024-11-27T00:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,146 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 199009045896d5ba974e92cd43d397ed in 472ms, sequenceid=85, compaction requested=false 2024-11-27T00:17:06,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:06,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-27T00:17:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-27T00:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-27T00:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-11-27T00:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0900 sec 2024-11-27T00:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T00:17:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,204 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,205 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-27T00:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,206 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T00:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,207 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:06,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T00:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T00:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-27T00:17:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-27T00:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-27T00:17:06,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-27T00:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 159 msec 2024-11-27T00:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:17:06,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:06,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:06,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:06,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:06,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:06,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127526cdc93a7474bf793d030fb3d5ded9e_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666626464/Put/seqid=0 2024-11-27T00:17:06,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742366_1542 (size=21918) 2024-11-27T00:17:06,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666626500, max timestamp=1732666626512Num uniques 859; 2024-11-27T00:17:06,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:06,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666626500, max timestamp=1732666626512Num uniques 859; 2024-11-27T00:17:06,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:06,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666626500, max timestamp=1732666626512Num uniques 859; 2024-11-27T00:17:06,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T00:17:06,524 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:06,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-27T00:17:06,526 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T00:17:06,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:06,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:06,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666686543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666686548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666686549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T00:17:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666686654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666686654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666686663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T00:17:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:06,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T00:17:06,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T00:17:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666686864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666686864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666686873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:06,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:06,890 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127526cdc93a7474bf793d030fb3d5ded9e_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127526cdc93a7474bf793d030fb3d5ded9e_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:06,891 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/a22f19cc391b4922a43908e12a0f5d9f, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:06,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/a22f19cc391b4922a43908e12a0f5d9f is 175, key is test_row_0/A:col10/1732666626464/Put/seqid=0 2024-11-27T00:17:06,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742367_1543 (size=65323) 2024-11-27T00:17:06,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T00:17:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:06,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T00:17:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T00:17:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666687174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666687174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:07,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666687184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:07,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T00:17:07,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:07,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,295 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/a22f19cc391b4922a43908e12a0f5d9f 2024-11-27T00:17:07,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/63492c53c92a4e9ba5afe00f846662af is 50, key is test_row_0/B:col10/1732666626464/Put/seqid=0 2024-11-27T00:17:07,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742368_1544 (size=12001) 2024-11-27T00:17:07,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/63492c53c92a4e9ba5afe00f846662af 2024-11-27T00:17:07,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/9c58b0b171474e588f7639ec55c197b0 is 50, key is test_row_0/C:col10/1732666626464/Put/seqid=0 2024-11-27T00:17:07,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742369_1545 (size=12001) 2024-11-27T00:17:07,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T00:17:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T00:17:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T00:17:07,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666687684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:07,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666687684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:07,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666687694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:07,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:07,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:07,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/9c58b0b171474e588f7639ec55c197b0 2024-11-27T00:17:07,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/a22f19cc391b4922a43908e12a0f5d9f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a22f19cc391b4922a43908e12a0f5d9f 2024-11-27T00:17:07,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a22f19cc391b4922a43908e12a0f5d9f, entries=350, sequenceid=96, filesize=63.8 K 2024-11-27T00:17:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/63492c53c92a4e9ba5afe00f846662af as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/63492c53c92a4e9ba5afe00f846662af 2024-11-27T00:17:07,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/63492c53c92a4e9ba5afe00f846662af, entries=150, sequenceid=96, filesize=11.7 K 2024-11-27T00:17:07,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/9c58b0b171474e588f7639ec55c197b0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9c58b0b171474e588f7639ec55c197b0 2024-11-27T00:17:07,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9c58b0b171474e588f7639ec55c197b0, entries=150, sequenceid=96, filesize=11.7 K 2024-11-27T00:17:07,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 199009045896d5ba974e92cd43d397ed in 1257ms, sequenceid=96, compaction requested=true 2024-11-27T00:17:07,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:07,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:07,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:07,733 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:07,734 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:07,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:07,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:07,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:17:07,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:07,734 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:07,734 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:07,734 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/A is initiating minor compaction (all files) 2024-11-27T00:17:07,734 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/B is initiating minor compaction (all files) 2024-11-27T00:17:07,734 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/A in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,734 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/B in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,734 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ddf73f6c26f04bbc8d438ff652c834b2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5fa3b9b9d50949209ecebe0a3181d8d3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a22f19cc391b4922a43908e12a0f5d9f] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=124.4 K 2024-11-27T00:17:07,734 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/05bab48d8868478281792e31a6a41f83, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5403256cdc3c490b83004339b93d2e94, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/63492c53c92a4e9ba5afe00f846662af] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=35.3 K 2024-11-27T00:17:07,734 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,734 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ddf73f6c26f04bbc8d438ff652c834b2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5fa3b9b9d50949209ecebe0a3181d8d3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a22f19cc391b4922a43908e12a0f5d9f] 2024-11-27T00:17:07,735 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf73f6c26f04bbc8d438ff652c834b2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732666624255 2024-11-27T00:17:07,735 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 05bab48d8868478281792e31a6a41f83, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732666624255 2024-11-27T00:17:07,735 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5403256cdc3c490b83004339b93d2e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732666624277 2024-11-27T00:17:07,735 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fa3b9b9d50949209ecebe0a3181d8d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732666624277 2024-11-27T00:17:07,735 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 63492c53c92a4e9ba5afe00f846662af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666626464 2024-11-27T00:17:07,735 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a22f19cc391b4922a43908e12a0f5d9f, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666626453 2024-11-27T00:17:07,739 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:07,741 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:07,741 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/f1f784667f9f4f7ca1ca18ff7cefece1 is 50, key is test_row_0/B:col10/1732666626464/Put/seqid=0 2024-11-27T00:17:07,741 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112734cfd0eb84a4492f9bd690f81ff847ec_199009045896d5ba974e92cd43d397ed store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:07,744 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112734cfd0eb84a4492f9bd690f81ff847ec_199009045896d5ba974e92cd43d397ed, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:07,744 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112734cfd0eb84a4492f9bd690f81ff847ec_199009045896d5ba974e92cd43d397ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:07,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T00:17:07,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,748 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T00:17:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:07,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742370_1546 (size=12207) 2024-11-27T00:17:07,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742371_1547 (size=4469) 2024-11-27T00:17:07,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277c273acd12cc453799198d58e4592c6c_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666626547/Put/seqid=0 2024-11-27T00:17:07,757 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#A#compaction#468 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:07,758 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/a946260054104508af9df04aa766d3d2 is 175, key is test_row_0/A:col10/1732666626464/Put/seqid=0 2024-11-27T00:17:07,762 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/f1f784667f9f4f7ca1ca18ff7cefece1 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f1f784667f9f4f7ca1ca18ff7cefece1 2024-11-27T00:17:07,767 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/B of 199009045896d5ba974e92cd43d397ed into f1f784667f9f4f7ca1ca18ff7cefece1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:07,767 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:07,767 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/B, priority=13, startTime=1732666627733; duration=0sec 2024-11-27T00:17:07,767 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:07,767 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:07,767 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:07,768 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:07,769 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/C is initiating minor compaction (all files) 2024-11-27T00:17:07,769 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/C in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:07,769 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/81ba8c22f61547a39e1e0234bcd5d825, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1a4ec1f11c5f48b18ee1169574549629, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9c58b0b171474e588f7639ec55c197b0] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=35.3 K 2024-11-27T00:17:07,769 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ba8c22f61547a39e1e0234bcd5d825, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732666624255 2024-11-27T00:17:07,770 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a4ec1f11c5f48b18ee1169574549629, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732666624277 2024-11-27T00:17:07,770 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c58b0b171474e588f7639ec55c197b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666626464 2024-11-27T00:17:07,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742373_1549 (size=31161) 2024-11-27T00:17:07,801 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#C#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:07,802 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/0135205be9804119873717a342f2ec83 is 50, key is test_row_0/C:col10/1732666626464/Put/seqid=0 2024-11-27T00:17:07,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742372_1548 (size=12254) 2024-11-27T00:17:07,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742374_1550 (size=12207) 2024-11-27T00:17:08,203 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/a946260054104508af9df04aa766d3d2 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a946260054104508af9df04aa766d3d2 2024-11-27T00:17:08,207 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/A of 199009045896d5ba974e92cd43d397ed into a946260054104508af9df04aa766d3d2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:08,207 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:08,207 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/A, priority=13, startTime=1732666627733; duration=0sec 2024-11-27T00:17:08,208 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:08,208 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:08,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,218 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277c273acd12cc453799198d58e4592c6c_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c273acd12cc453799198d58e4592c6c_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:08,219 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/0135205be9804119873717a342f2ec83 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/0135205be9804119873717a342f2ec83 2024-11-27T00:17:08,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/5985519833db4597b3667643e42ad022, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:08,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/5985519833db4597b3667643e42ad022 is 175, key is test_row_0/A:col10/1732666626547/Put/seqid=0 2024-11-27T00:17:08,224 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/C of 199009045896d5ba974e92cd43d397ed into 0135205be9804119873717a342f2ec83(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:08,224 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:08,224 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/C, priority=13, startTime=1732666627734; duration=0sec 2024-11-27T00:17:08,224 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:08,224 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:08,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742375_1551 (size=31055) 2024-11-27T00:17:08,225 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/5985519833db4597b3667643e42ad022 2024-11-27T00:17:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/8feebca6c61e46a6bdd3e4c76aa56bcc is 50, key is test_row_0/B:col10/1732666626547/Put/seqid=0 2024-11-27T00:17:08,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742376_1552 (size=12101) 2024-11-27T00:17:08,250 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/8feebca6c61e46a6bdd3e4c76aa56bcc 2024-11-27T00:17:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/d84c8dd714d945b3b0d51731b3050a62 is 50, key is test_row_0/C:col10/1732666626547/Put/seqid=0 2024-11-27T00:17:08,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742377_1553 (size=12101) 2024-11-27T00:17:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T00:17:08,692 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/d84c8dd714d945b3b0d51731b3050a62 2024-11-27T00:17:08,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/5985519833db4597b3667643e42ad022 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5985519833db4597b3667643e42ad022 2024-11-27T00:17:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:08,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:08,698 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5985519833db4597b3667643e42ad022, entries=150, sequenceid=132, filesize=30.3 K 2024-11-27T00:17:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/8feebca6c61e46a6bdd3e4c76aa56bcc as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/8feebca6c61e46a6bdd3e4c76aa56bcc 2024-11-27T00:17:08,702 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/8feebca6c61e46a6bdd3e4c76aa56bcc, entries=150, sequenceid=132, filesize=11.8 K 2024-11-27T00:17:08,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/d84c8dd714d945b3b0d51731b3050a62 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d84c8dd714d945b3b0d51731b3050a62 2024-11-27T00:17:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,705 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d84c8dd714d945b3b0d51731b3050a62, entries=150, sequenceid=132, filesize=11.8 K 2024-11-27T00:17:08,706 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=40.25 KB/41220 for 199009045896d5ba974e92cd43d397ed in 958ms, sequenceid=132, compaction requested=false 2024-11-27T00:17:08,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:08,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:08,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-27T00:17:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-27T00:17:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-27T00:17:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-27T00:17:08,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-11-27T00:17:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1850 sec 2024-11-27T00:17:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127afd9e82d69eb43c490d855eaf00a168a_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666628707/Put/seqid=0 2024-11-27T00:17:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742378_1554 (size=17284) 2024-11-27T00:17:08,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:08,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127afd9e82d69eb43c490d855eaf00a168a_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127afd9e82d69eb43c490d855eaf00a168a_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:08,740 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9d770b0e9b924a5bba467bfff94ecfa3, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:08,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9d770b0e9b924a5bba467bfff94ecfa3 is 175, key is test_row_0/A:col10/1732666628707/Put/seqid=0 2024-11-27T00:17:08,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742379_1555 (size=48389) 2024-11-27T00:17:08,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9d770b0e9b924a5bba467bfff94ecfa3 2024-11-27T00:17:08,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/344620b4b0ea44fbba9b1bb65ede621d is 50, key is test_row_0/B:col10/1732666628707/Put/seqid=0 2024-11-27T00:17:08,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666688755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:08,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666628734, max timestamp=1732666628758Num uniques 859; 2024-11-27T00:17:08,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:08,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666628734, max timestamp=1732666628758Num uniques 859; 2024-11-27T00:17:08,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732666628734, max timestamp=1732666628758Num uniques 859; 2024-11-27T00:17:08,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:08,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f036bc4986ea:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-27T00:17:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:08,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742380_1556 (size=12151) 2024-11-27T00:17:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666688802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:08,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666688808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666688864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:08,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666688914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:08,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666688924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666689074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666689124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666689134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/344620b4b0ea44fbba9b1bb65ede621d 2024-11-27T00:17:09,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/ffacd8d6fe464b839c1e5237c4f36575 is 50, key is test_row_0/C:col10/1732666628707/Put/seqid=0 2024-11-27T00:17:09,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666689184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,188 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666689194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,199 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742381_1557 (size=12151) 2024-11-27T00:17:09,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/ffacd8d6fe464b839c1e5237c4f36575 2024-11-27T00:17:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9d770b0e9b924a5bba467bfff94ecfa3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9d770b0e9b924a5bba467bfff94ecfa3 2024-11-27T00:17:09,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9d770b0e9b924a5bba467bfff94ecfa3, entries=250, sequenceid=146, filesize=47.3 K 2024-11-27T00:17:09,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/344620b4b0ea44fbba9b1bb65ede621d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/344620b4b0ea44fbba9b1bb65ede621d 2024-11-27T00:17:09,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/344620b4b0ea44fbba9b1bb65ede621d, entries=150, sequenceid=146, filesize=11.9 K 2024-11-27T00:17:09,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/ffacd8d6fe464b839c1e5237c4f36575 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ffacd8d6fe464b839c1e5237c4f36575 2024-11-27T00:17:09,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ffacd8d6fe464b839c1e5237c4f36575, entries=150, sequenceid=146, filesize=11.9 K 2024-11-27T00:17:09,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 199009045896d5ba974e92cd43d397ed in 529ms, sequenceid=146, compaction requested=true 2024-11-27T00:17:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:09,237 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:09,237 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:17:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:09,238 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:09,238 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/A is initiating minor compaction (all files) 2024-11-27T00:17:09,238 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/A in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:09,238 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a946260054104508af9df04aa766d3d2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5985519833db4597b3667643e42ad022, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9d770b0e9b924a5bba467bfff94ecfa3] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=108.0 K 2024-11-27T00:17:09,238 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:09,238 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a946260054104508af9df04aa766d3d2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5985519833db4597b3667643e42ad022, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9d770b0e9b924a5bba467bfff94ecfa3] 2024-11-27T00:17:09,239 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a946260054104508af9df04aa766d3d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666626464 2024-11-27T00:17:09,239 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:09,239 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/B is initiating minor compaction (all files) 2024-11-27T00:17:09,239 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/B in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:09,239 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f1f784667f9f4f7ca1ca18ff7cefece1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/8feebca6c61e46a6bdd3e4c76aa56bcc, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/344620b4b0ea44fbba9b1bb65ede621d] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=35.6 K 2024-11-27T00:17:09,239 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5985519833db4597b3667643e42ad022, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732666626503 2024-11-27T00:17:09,239 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f784667f9f4f7ca1ca18ff7cefece1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666626464 2024-11-27T00:17:09,239 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d770b0e9b924a5bba467bfff94ecfa3, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732666628698 2024-11-27T00:17:09,240 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8feebca6c61e46a6bdd3e4c76aa56bcc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732666626503 2024-11-27T00:17:09,240 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 344620b4b0ea44fbba9b1bb65ede621d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732666628698 2024-11-27T00:17:09,256 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#B#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:09,256 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/14d77c5513af43a4965ec17029cdc427 is 50, key is test_row_0/B:col10/1732666628707/Put/seqid=0 2024-11-27T00:17:09,272 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:09,297 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127f70968551b61488bb25b6eebadc94fc6_199009045896d5ba974e92cd43d397ed store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:09,299 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127f70968551b61488bb25b6eebadc94fc6_199009045896d5ba974e92cd43d397ed, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:09,299 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f70968551b61488bb25b6eebadc94fc6_199009045896d5ba974e92cd43d397ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:09,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742382_1558 (size=12459) 2024-11-27T00:17:09,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742383_1559 (size=4469) 2024-11-27T00:17:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:09,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-27T00:17:09,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:09,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:09,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:09,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:09,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:09,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-27T00:17:09,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274f487fdfb8314a329ea8fe81953c2009_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666628799/Put/seqid=0 2024-11-27T00:17:09,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742384_1560 (size=14794) 2024-11-27T00:17:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666689434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666689441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666689443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666689553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,728 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/14d77c5513af43a4965ec17029cdc427 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/14d77c5513af43a4965ec17029cdc427 2024-11-27T00:17:09,733 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/B of 199009045896d5ba974e92cd43d397ed into 14d77c5513af43a4965ec17029cdc427(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:09,733 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:09,733 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/B, priority=13, startTime=1732666629237; duration=0sec 2024-11-27T00:17:09,733 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:09,733 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:09,734 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:09,736 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:09,736 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/C is initiating minor compaction (all files) 2024-11-27T00:17:09,736 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/C in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:09,736 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/0135205be9804119873717a342f2ec83, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d84c8dd714d945b3b0d51731b3050a62, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ffacd8d6fe464b839c1e5237c4f36575] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=35.6 K 2024-11-27T00:17:09,736 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0135205be9804119873717a342f2ec83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732666626464 2024-11-27T00:17:09,737 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d84c8dd714d945b3b0d51731b3050a62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732666626503 2024-11-27T00:17:09,737 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ffacd8d6fe464b839c1e5237c4f36575, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732666628698 2024-11-27T00:17:09,746 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:09,747 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/09bae8e7ddc24efbbb6294eae6b7b882 is 50, key is test_row_0/C:col10/1732666628707/Put/seqid=0 2024-11-27T00:17:09,752 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#A#compaction#477 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:09,753 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9442071aa0514933838ed65f9f8dc127 is 175, key is test_row_0/A:col10/1732666628707/Put/seqid=0 2024-11-27T00:17:09,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666689764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:09,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742385_1561 (size=12459) 2024-11-27T00:17:09,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742386_1562 (size=31413) 2024-11-27T00:17:09,810 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:09,814 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274f487fdfb8314a329ea8fe81953c2009_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274f487fdfb8314a329ea8fe81953c2009_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:09,815 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/8eabba9d1580421e9b946c2138197301, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:09,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/8eabba9d1580421e9b946c2138197301 is 175, key is test_row_0/A:col10/1732666628799/Put/seqid=0 2024-11-27T00:17:09,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742387_1563 (size=39749) 2024-11-27T00:17:09,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/8eabba9d1580421e9b946c2138197301 2024-11-27T00:17:09,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/6828083ccd7e4a4da7db89407d4bfb90 is 50, key is test_row_0/B:col10/1732666628799/Put/seqid=0 2024-11-27T00:17:09,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742388_1564 (size=12151) 2024-11-27T00:17:09,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/6828083ccd7e4a4da7db89407d4bfb90 2024-11-27T00:17:09,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4d2c6afcc66244c98821ea1126ece7a3 is 50, key is test_row_0/C:col10/1732666628799/Put/seqid=0 2024-11-27T00:17:09,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742389_1565 (size=12151) 2024-11-27T00:17:09,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4d2c6afcc66244c98821ea1126ece7a3 2024-11-27T00:17:09,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/8eabba9d1580421e9b946c2138197301 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/8eabba9d1580421e9b946c2138197301 2024-11-27T00:17:09,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/8eabba9d1580421e9b946c2138197301, entries=200, sequenceid=182, filesize=38.8 K 2024-11-27T00:17:09,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/6828083ccd7e4a4da7db89407d4bfb90 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6828083ccd7e4a4da7db89407d4bfb90 2024-11-27T00:17:09,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6828083ccd7e4a4da7db89407d4bfb90, entries=150, sequenceid=182, filesize=11.9 K 2024-11-27T00:17:09,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4d2c6afcc66244c98821ea1126ece7a3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4d2c6afcc66244c98821ea1126ece7a3 2024-11-27T00:17:09,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4d2c6afcc66244c98821ea1126ece7a3, entries=150, sequenceid=182, filesize=11.9 K 2024-11-27T00:17:09,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 199009045896d5ba974e92cd43d397ed in 553ms, sequenceid=182, compaction requested=false 2024-11-27T00:17:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:09,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-27T00:17:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:09,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127917ac97c90304b4f8ea344ffe8618036_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666629958/Put/seqid=0 2024-11-27T00:17:10,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742390_1566 (size=14794) 2024-11-27T00:17:10,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666690041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666690044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666690074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666690154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666690154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,195 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/09bae8e7ddc24efbbb6294eae6b7b882 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/09bae8e7ddc24efbbb6294eae6b7b882 2024-11-27T00:17:10,200 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/C of 199009045896d5ba974e92cd43d397ed into 09bae8e7ddc24efbbb6294eae6b7b882(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:10,201 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:10,201 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/C, priority=13, startTime=1732666629237; duration=0sec 2024-11-27T00:17:10,201 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:10,201 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:10,214 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9442071aa0514933838ed65f9f8dc127 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9442071aa0514933838ed65f9f8dc127 2024-11-27T00:17:10,227 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/A of 199009045896d5ba974e92cd43d397ed into 9442071aa0514933838ed65f9f8dc127(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:10,229 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:10,229 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/A, priority=13, startTime=1732666629237; duration=0sec 2024-11-27T00:17:10,229 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:10,229 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:10,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666690364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666690364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,425 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:10,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127917ac97c90304b4f8ea344ffe8618036_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127917ac97c90304b4f8ea344ffe8618036_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:10,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/6721e72659db4585925285054aea881e, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:10,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/6721e72659db4585925285054aea881e is 175, key is test_row_0/A:col10/1732666629958/Put/seqid=0 2024-11-27T00:17:10,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742391_1567 (size=39749) 2024-11-27T00:17:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666690584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666690674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666690674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T00:17:10,684 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:10,685 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-27T00:17:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T00:17:10,686 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:10,687 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:10,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T00:17:10,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T00:17:10,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:10,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:10,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:10,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:10,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:10,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/6721e72659db4585925285054aea881e 2024-11-27T00:17:10,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/147b1908fce441639383d77193f863dd is 50, key is test_row_0/B:col10/1732666629958/Put/seqid=0 2024-11-27T00:17:10,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742392_1568 (size=12151) 2024-11-27T00:17:10,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/147b1908fce441639383d77193f863dd 2024-11-27T00:17:10,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4e873e954026498197ef67a341bd7130 is 50, key is test_row_0/C:col10/1732666629958/Put/seqid=0 2024-11-27T00:17:10,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742393_1569 (size=12151) 2024-11-27T00:17:10,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4e873e954026498197ef67a341bd7130 2024-11-27T00:17:10,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/6721e72659db4585925285054aea881e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/6721e72659db4585925285054aea881e 2024-11-27T00:17:10,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/6721e72659db4585925285054aea881e, entries=200, sequenceid=195, filesize=38.8 K 2024-11-27T00:17:10,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/147b1908fce441639383d77193f863dd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/147b1908fce441639383d77193f863dd 2024-11-27T00:17:10,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/147b1908fce441639383d77193f863dd, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T00:17:10,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4e873e954026498197ef67a341bd7130 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4e873e954026498197ef67a341bd7130 2024-11-27T00:17:10,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4e873e954026498197ef67a341bd7130, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T00:17:10,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 199009045896d5ba974e92cd43d397ed in 950ms, sequenceid=195, compaction requested=true 2024-11-27T00:17:10,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:10,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:10,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:10,909 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:10,909 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:10,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:10,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:10,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:17:10,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:10,911 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:10,911 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/B is initiating minor compaction (all files) 2024-11-27T00:17:10,911 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/B in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:10,911 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/14d77c5513af43a4965ec17029cdc427, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6828083ccd7e4a4da7db89407d4bfb90, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/147b1908fce441639383d77193f863dd] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=35.9 K 2024-11-27T00:17:10,911 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:10,911 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/A is initiating minor compaction (all files) 2024-11-27T00:17:10,911 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/A in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:10,911 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9442071aa0514933838ed65f9f8dc127, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/8eabba9d1580421e9b946c2138197301, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/6721e72659db4585925285054aea881e] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=108.3 K 2024-11-27T00:17:10,911 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:10,911 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9442071aa0514933838ed65f9f8dc127, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/8eabba9d1580421e9b946c2138197301, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/6721e72659db4585925285054aea881e] 2024-11-27T00:17:10,912 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d77c5513af43a4965ec17029cdc427, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732666628698 2024-11-27T00:17:10,912 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9442071aa0514933838ed65f9f8dc127, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732666628698 2024-11-27T00:17:10,912 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6828083ccd7e4a4da7db89407d4bfb90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732666628799 2024-11-27T00:17:10,912 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eabba9d1580421e9b946c2138197301, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732666628799 2024-11-27T00:17:10,912 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6721e72659db4585925285054aea881e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732666629423 2024-11-27T00:17:10,912 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 147b1908fce441639383d77193f863dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732666629423 2024-11-27T00:17:10,919 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:10,920 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#B#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:10,921 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/91be6451d8d04cf680d9b7566f4c275d is 50, key is test_row_0/B:col10/1732666629958/Put/seqid=0 2024-11-27T00:17:10,923 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112766232d9d4c7542a6afb3b12debef08be_199009045896d5ba974e92cd43d397ed store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:10,925 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112766232d9d4c7542a6afb3b12debef08be_199009045896d5ba974e92cd43d397ed, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:10,926 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112766232d9d4c7542a6afb3b12debef08be_199009045896d5ba974e92cd43d397ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:10,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742394_1570 (size=12561) 2024-11-27T00:17:10,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742395_1571 (size=4469) 2024-11-27T00:17:10,975 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#A#compaction#486 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:10,976 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/30583206c6fe402ca82b36341a6ab911 is 175, key is test_row_0/A:col10/1732666629958/Put/seqid=0 2024-11-27T00:17:10,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T00:17:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:10,993 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-27T00:17:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:10,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:10,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742396_1572 (size=31515) 2024-11-27T00:17:11,004 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/30583206c6fe402ca82b36341a6ab911 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/30583206c6fe402ca82b36341a6ab911 2024-11-27T00:17:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T00:17:11,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278f96da85219d4a12b0555bbc7ed48194_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666630042/Put/seqid=0 2024-11-27T00:17:11,011 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/A of 199009045896d5ba974e92cd43d397ed into 30583206c6fe402ca82b36341a6ab911(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:11,011 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:11,011 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/A, priority=13, startTime=1732666630909; duration=0sec 2024-11-27T00:17:11,011 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:11,011 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:11,012 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:11,013 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:11,013 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/C is initiating minor compaction (all files) 2024-11-27T00:17:11,013 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/C in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:11,013 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/09bae8e7ddc24efbbb6294eae6b7b882, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4d2c6afcc66244c98821ea1126ece7a3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4e873e954026498197ef67a341bd7130] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=35.9 K 2024-11-27T00:17:11,013 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09bae8e7ddc24efbbb6294eae6b7b882, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732666628698 2024-11-27T00:17:11,014 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d2c6afcc66244c98821ea1126ece7a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732666628799 2024-11-27T00:17:11,014 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e873e954026498197ef67a341bd7130, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732666629423 2024-11-27T00:17:11,030 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:11,031 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/434fc1a2cc7f47f691face04bc0e6e8a is 50, key is test_row_0/C:col10/1732666629958/Put/seqid=0 2024-11-27T00:17:11,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742397_1573 (size=12304) 2024-11-27T00:17:11,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742398_1574 (size=12561) 2024-11-27T00:17:11,075 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/434fc1a2cc7f47f691face04bc0e6e8a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/434fc1a2cc7f47f691face04bc0e6e8a 2024-11-27T00:17:11,082 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/C of 199009045896d5ba974e92cd43d397ed into 434fc1a2cc7f47f691face04bc0e6e8a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:11,082 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:11,082 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/C, priority=13, startTime=1732666630909; duration=0sec 2024-11-27T00:17:11,082 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:11,082 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:11,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:11,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666691211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666691211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T00:17:11,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666691324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666691324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,336 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/91be6451d8d04cf680d9b7566f4c275d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/91be6451d8d04cf680d9b7566f4c275d 2024-11-27T00:17:11,342 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/B of 199009045896d5ba974e92cd43d397ed into 91be6451d8d04cf680d9b7566f4c275d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:11,342 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:11,342 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/B, priority=13, startTime=1732666630909; duration=0sec 2024-11-27T00:17:11,343 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:11,343 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:11,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:11,455 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278f96da85219d4a12b0555bbc7ed48194_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278f96da85219d4a12b0555bbc7ed48194_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:11,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/4e693b35f51a48a595e9fd402bf122f7, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/4e693b35f51a48a595e9fd402bf122f7 is 175, key is test_row_0/A:col10/1732666630042/Put/seqid=0 2024-11-27T00:17:11,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742399_1575 (size=31105) 2024-11-27T00:17:11,465 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/4e693b35f51a48a595e9fd402bf122f7 2024-11-27T00:17:11,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/5cb344dfc5124074bb7ca978f2287eab is 50, key is test_row_0/B:col10/1732666630042/Put/seqid=0 2024-11-27T00:17:11,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742400_1576 (size=12151) 2024-11-27T00:17:11,478 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/5cb344dfc5124074bb7ca978f2287eab 2024-11-27T00:17:11,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/5aa514c82ea74dcba02dedbee8892e1e is 50, key is test_row_0/C:col10/1732666630042/Put/seqid=0 2024-11-27T00:17:11,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742401_1577 (size=12151) 2024-11-27T00:17:11,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666691533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666691533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666691604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,607 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T00:17:11,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666691844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666691844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:11,890 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/5aa514c82ea74dcba02dedbee8892e1e 2024-11-27T00:17:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/4e693b35f51a48a595e9fd402bf122f7 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e693b35f51a48a595e9fd402bf122f7 2024-11-27T00:17:11,897 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e693b35f51a48a595e9fd402bf122f7, entries=150, sequenceid=217, filesize=30.4 K 2024-11-27T00:17:11,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/5cb344dfc5124074bb7ca978f2287eab as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5cb344dfc5124074bb7ca978f2287eab 2024-11-27T00:17:11,900 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5cb344dfc5124074bb7ca978f2287eab, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T00:17:11,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/5aa514c82ea74dcba02dedbee8892e1e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5aa514c82ea74dcba02dedbee8892e1e 2024-11-27T00:17:11,903 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5aa514c82ea74dcba02dedbee8892e1e, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T00:17:11,903 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 199009045896d5ba974e92cd43d397ed in 910ms, sequenceid=217, compaction requested=false 2024-11-27T00:17:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-27T00:17:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-27T00:17:11,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-27T00:17:11,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2170 sec 2024-11-27T00:17:11,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.2210 sec 2024-11-27T00:17:12,122 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T00:17:12,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-27T00:17:12,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:12,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127be96ef9c31734e6d9610888c8a26bbc9_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666632357/Put/seqid=0 2024-11-27T00:17:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666692408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:12,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742402_1578 (size=12304) 2024-11-27T00:17:12,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666692414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666692523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666692524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666692734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666692734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:12,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:12,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127be96ef9c31734e6d9610888c8a26bbc9_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127be96ef9c31734e6d9610888c8a26bbc9_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:12,822 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/260f61870f3842a99c56dac085ced5a0, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:12,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/260f61870f3842a99c56dac085ced5a0 is 175, key is test_row_0/A:col10/1732666632357/Put/seqid=0 2024-11-27T00:17:12,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742403_1579 (size=31105) 2024-11-27T00:17:12,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/260f61870f3842a99c56dac085ced5a0 2024-11-27T00:17:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T00:17:12,835 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:12,835 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:12,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/e1bd0a100691484e82b63fd071c60c55 is 50, key is test_row_0/B:col10/1732666632357/Put/seqid=0 2024-11-27T00:17:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-27T00:17:12,837 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T00:17:12,838 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:12,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:12,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742404_1580 (size=12151) 2024-11-27T00:17:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T00:17:12,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T00:17:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:12,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666693044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666693044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T00:17:13,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:13,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:13,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:13,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:13,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T00:17:13,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/e1bd0a100691484e82b63fd071c60c55 2024-11-27T00:17:13,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/33e1938613c243b8bdabfb1d02b972a9 is 50, key is test_row_0/C:col10/1732666632357/Put/seqid=0 2024-11-27T00:17:13,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742405_1581 (size=12151) 2024-11-27T00:17:13,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/33e1938613c243b8bdabfb1d02b972a9 2024-11-27T00:17:13,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/260f61870f3842a99c56dac085ced5a0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/260f61870f3842a99c56dac085ced5a0 2024-11-27T00:17:13,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/260f61870f3842a99c56dac085ced5a0, entries=150, sequenceid=235, filesize=30.4 K 2024-11-27T00:17:13,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/e1bd0a100691484e82b63fd071c60c55 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/e1bd0a100691484e82b63fd071c60c55 2024-11-27T00:17:13,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/e1bd0a100691484e82b63fd071c60c55, entries=150, sequenceid=235, filesize=11.9 K 2024-11-27T00:17:13,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/33e1938613c243b8bdabfb1d02b972a9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/33e1938613c243b8bdabfb1d02b972a9 2024-11-27T00:17:13,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/33e1938613c243b8bdabfb1d02b972a9, entries=150, sequenceid=235, filesize=11.9 K 2024-11-27T00:17:13,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 199009045896d5ba974e92cd43d397ed in 919ms, sequenceid=235, compaction requested=true 2024-11-27T00:17:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:13,278 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:13,278 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:17:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:13,279 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:13,279 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:13,279 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/A is initiating minor compaction (all files) 2024-11-27T00:17:13,279 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/B is initiating minor compaction (all files) 2024-11-27T00:17:13,279 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/B in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:13,279 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/A in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:13,279 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/91be6451d8d04cf680d9b7566f4c275d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5cb344dfc5124074bb7ca978f2287eab, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/e1bd0a100691484e82b63fd071c60c55] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=36.0 K 2024-11-27T00:17:13,279 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/30583206c6fe402ca82b36341a6ab911, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e693b35f51a48a595e9fd402bf122f7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/260f61870f3842a99c56dac085ced5a0] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=91.5 K 2024-11-27T00:17:13,279 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:13,279 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/30583206c6fe402ca82b36341a6ab911, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e693b35f51a48a595e9fd402bf122f7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/260f61870f3842a99c56dac085ced5a0] 2024-11-27T00:17:13,279 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 91be6451d8d04cf680d9b7566f4c275d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732666629423 2024-11-27T00:17:13,280 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb344dfc5124074bb7ca978f2287eab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732666630034 2024-11-27T00:17:13,280 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30583206c6fe402ca82b36341a6ab911, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732666629423 2024-11-27T00:17:13,280 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e1bd0a100691484e82b63fd071c60c55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666631195 2024-11-27T00:17:13,280 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e693b35f51a48a595e9fd402bf122f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732666630034 2024-11-27T00:17:13,281 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260f61870f3842a99c56dac085ced5a0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666631195 2024-11-27T00:17:13,289 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:13,290 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:13,291 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/be6cc36ac0c74d87a4d8234d5971ad70 is 50, key is test_row_0/B:col10/1732666632357/Put/seqid=0 2024-11-27T00:17:13,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-27T00:17:13,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:13,295 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:17:13,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:13,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:13,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:13,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:13,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:13,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:13,300 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411271f23840d38984888aad9f5a1322c6e2b_199009045896d5ba974e92cd43d397ed store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:13,302 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411271f23840d38984888aad9f5a1322c6e2b_199009045896d5ba974e92cd43d397ed, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:13,303 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271f23840d38984888aad9f5a1322c6e2b_199009045896d5ba974e92cd43d397ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:13,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276f07d0a2e1464439b47fd53848fc100a_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666632411/Put/seqid=0 2024-11-27T00:17:13,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742406_1582 (size=12663) 2024-11-27T00:17:13,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742407_1583 (size=4469) 2024-11-27T00:17:13,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742408_1584 (size=12304) 2024-11-27T00:17:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T00:17:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:13,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666693587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666693591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666693634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,639 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666693693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666693704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,743 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/be6cc36ac0c74d87a4d8234d5971ad70 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/be6cc36ac0c74d87a4d8234d5971ad70 2024-11-27T00:17:13,748 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/B of 199009045896d5ba974e92cd43d397ed into be6cc36ac0c74d87a4d8234d5971ad70(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:13,748 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:13,748 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/B, priority=13, startTime=1732666633278; duration=0sec 2024-11-27T00:17:13,749 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:13,749 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:13,749 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:13,750 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:13,750 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/C is initiating minor compaction (all files) 2024-11-27T00:17:13,750 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/C in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:13,750 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/434fc1a2cc7f47f691face04bc0e6e8a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5aa514c82ea74dcba02dedbee8892e1e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/33e1938613c243b8bdabfb1d02b972a9] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=36.0 K 2024-11-27T00:17:13,750 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 434fc1a2cc7f47f691face04bc0e6e8a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732666629423 2024-11-27T00:17:13,751 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aa514c82ea74dcba02dedbee8892e1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732666630034 2024-11-27T00:17:13,752 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#A#compaction#495 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:13,753 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/66dcf346912943d29779b7ad20ccf409 is 175, key is test_row_0/A:col10/1732666632357/Put/seqid=0 2024-11-27T00:17:13,754 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 33e1938613c243b8bdabfb1d02b972a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666631195 2024-11-27T00:17:13,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:13,766 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276f07d0a2e1464439b47fd53848fc100a_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276f07d0a2e1464439b47fd53848fc100a_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:13,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/930c854fa97c4602a0eb9799e4625319, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:13,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/930c854fa97c4602a0eb9799e4625319 is 175, key is test_row_0/A:col10/1732666632411/Put/seqid=0 2024-11-27T00:17:13,775 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:13,775 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/d25a64c1f5e148b8afe691aff054928c is 50, key is test_row_0/C:col10/1732666632357/Put/seqid=0 2024-11-27T00:17:13,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742409_1585 (size=31617) 2024-11-27T00:17:13,800 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/66dcf346912943d29779b7ad20ccf409 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/66dcf346912943d29779b7ad20ccf409 2024-11-27T00:17:13,811 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/A of 199009045896d5ba974e92cd43d397ed into 66dcf346912943d29779b7ad20ccf409(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:13,811 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:13,811 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/A, priority=13, startTime=1732666633278; duration=0sec 2024-11-27T00:17:13,811 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:13,811 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:13,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742410_1586 (size=31105) 2024-11-27T00:17:13,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742411_1587 (size=12663) 2024-11-27T00:17:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666693903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666693914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T00:17:14,214 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/930c854fa97c4602a0eb9799e4625319 2024-11-27T00:17:14,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666694214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:14,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/45202a6f3e0c4b16982a25c062ceb9e3 is 50, key is test_row_0/B:col10/1732666632411/Put/seqid=0 2024-11-27T00:17:14,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742412_1588 (size=12151) 2024-11-27T00:17:14,223 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/45202a6f3e0c4b16982a25c062ceb9e3 2024-11-27T00:17:14,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666694224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/ead817fa39b143699133c0066a58c190 is 50, key is test_row_0/C:col10/1732666632411/Put/seqid=0 2024-11-27T00:17:14,240 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/d25a64c1f5e148b8afe691aff054928c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d25a64c1f5e148b8afe691aff054928c 2024-11-27T00:17:14,244 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/C of 199009045896d5ba974e92cd43d397ed into d25a64c1f5e148b8afe691aff054928c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:14,245 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:14,245 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/C, priority=13, startTime=1732666633278; duration=0sec 2024-11-27T00:17:14,245 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:14,245 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:14,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742413_1589 (size=12151) 2024-11-27T00:17:14,245 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/ead817fa39b143699133c0066a58c190 2024-11-27T00:17:14,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/930c854fa97c4602a0eb9799e4625319 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/930c854fa97c4602a0eb9799e4625319 2024-11-27T00:17:14,256 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/930c854fa97c4602a0eb9799e4625319, entries=150, sequenceid=252, filesize=30.4 K 2024-11-27T00:17:14,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/45202a6f3e0c4b16982a25c062ceb9e3 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/45202a6f3e0c4b16982a25c062ceb9e3 2024-11-27T00:17:14,259 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/45202a6f3e0c4b16982a25c062ceb9e3, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T00:17:14,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/ead817fa39b143699133c0066a58c190 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ead817fa39b143699133c0066a58c190 2024-11-27T00:17:14,265 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ead817fa39b143699133c0066a58c190, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T00:17:14,266 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 199009045896d5ba974e92cd43d397ed in 971ms, sequenceid=252, compaction requested=false 2024-11-27T00:17:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-27T00:17:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-27T00:17:14,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-27T00:17:14,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-27T00:17:14,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.4340 sec 2024-11-27T00:17:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:14,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:17:14,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:14,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:14,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:14,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:14,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:14,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:14,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272704330a897d4db2910a7bdc481083f3_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666633589/Put/seqid=0 2024-11-27T00:17:14,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742414_1590 (size=14994) 2024-11-27T00:17:14,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:14,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272704330a897d4db2910a7bdc481083f3_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272704330a897d4db2910a7bdc481083f3_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:14,740 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/70869a8b75b844ef9803970cfb5ad49a, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:14,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/70869a8b75b844ef9803970cfb5ad49a is 175, key is test_row_0/A:col10/1732666633589/Put/seqid=0 2024-11-27T00:17:14,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742415_1591 (size=39949) 2024-11-27T00:17:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666694775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666694776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:14,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666694884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:14,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666694884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T00:17:14,984 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:14,984 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-27T00:17:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T00:17:14,986 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:14,987 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:14,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T00:17:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666695093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666695094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:15,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T00:17:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,144 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/70869a8b75b844ef9803970cfb5ad49a 2024-11-27T00:17:15,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/73ea5f276c104463bd8efe770818f7a0 is 50, key is test_row_0/B:col10/1732666633589/Put/seqid=0 2024-11-27T00:17:15,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742416_1592 (size=12301) 2024-11-27T00:17:15,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T00:17:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:15,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T00:17:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666695404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666695404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:15,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T00:17:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/73ea5f276c104463bd8efe770818f7a0 2024-11-27T00:17:15,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T00:17:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/c1fa4065121c46768cd7601729565acf is 50, key is test_row_0/C:col10/1732666633589/Put/seqid=0 2024-11-27T00:17:15,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742417_1593 (size=12301) 2024-11-27T00:17:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T00:17:15,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T00:17:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T00:17:15,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:15,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:15,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:15,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666695914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:15,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666695914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:16,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/c1fa4065121c46768cd7601729565acf 2024-11-27T00:17:16,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/70869a8b75b844ef9803970cfb5ad49a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/70869a8b75b844ef9803970cfb5ad49a 2024-11-27T00:17:16,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/70869a8b75b844ef9803970cfb5ad49a, entries=200, sequenceid=271, filesize=39.0 K 2024-11-27T00:17:16,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/73ea5f276c104463bd8efe770818f7a0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/73ea5f276c104463bd8efe770818f7a0 2024-11-27T00:17:16,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/73ea5f276c104463bd8efe770818f7a0, entries=150, sequenceid=271, filesize=12.0 K 2024-11-27T00:17:16,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/c1fa4065121c46768cd7601729565acf as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/c1fa4065121c46768cd7601729565acf 2024-11-27T00:17:16,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/c1fa4065121c46768cd7601729565acf, entries=150, sequenceid=271, filesize=12.0 K 2024-11-27T00:17:16,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 199009045896d5ba974e92cd43d397ed in 1302ms, sequenceid=271, compaction requested=true 2024-11-27T00:17:16,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:16,031 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:16,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:16,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:16,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:16,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:16,031 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:16,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:17:16,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:16,031 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:16,031 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/A is initiating minor compaction (all files) 2024-11-27T00:17:16,032 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/A in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:16,032 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/66dcf346912943d29779b7ad20ccf409, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/930c854fa97c4602a0eb9799e4625319, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/70869a8b75b844ef9803970cfb5ad49a] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=100.3 K 2024-11-27T00:17:16,032 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:16,032 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/66dcf346912943d29779b7ad20ccf409, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/930c854fa97c4602a0eb9799e4625319, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/70869a8b75b844ef9803970cfb5ad49a] 2024-11-27T00:17:16,032 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:16,032 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66dcf346912943d29779b7ad20ccf409, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666631195 2024-11-27T00:17:16,032 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/B is initiating minor compaction (all files) 2024-11-27T00:17:16,032 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/B in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:16,032 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/be6cc36ac0c74d87a4d8234d5971ad70, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/45202a6f3e0c4b16982a25c062ceb9e3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/73ea5f276c104463bd8efe770818f7a0] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=36.2 K 2024-11-27T00:17:16,032 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 930c854fa97c4602a0eb9799e4625319, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732666632397 2024-11-27T00:17:16,032 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting be6cc36ac0c74d87a4d8234d5971ad70, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666631195 2024-11-27T00:17:16,032 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70869a8b75b844ef9803970cfb5ad49a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732666633579 2024-11-27T00:17:16,033 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 45202a6f3e0c4b16982a25c062ceb9e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732666632397 2024-11-27T00:17:16,033 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ea5f276c104463bd8efe770818f7a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732666633585 2024-11-27T00:17:16,039 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#B#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:16,039 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/6693b354fb91413caab8a05ed4425c98 is 50, key is test_row_0/B:col10/1732666633589/Put/seqid=0 2024-11-27T00:17:16,041 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:16,043 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d1f6c42ca8c549a1939560a6f707ea01_199009045896d5ba974e92cd43d397ed store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:16,045 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d1f6c42ca8c549a1939560a6f707ea01_199009045896d5ba974e92cd43d397ed, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:16,045 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d1f6c42ca8c549a1939560a6f707ea01_199009045896d5ba974e92cd43d397ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:16,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742418_1594 (size=12915) 2024-11-27T00:17:16,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742419_1595 (size=4469) 2024-11-27T00:17:16,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-27T00:17:16,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:16,059 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:17:16,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:16,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:16,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:16,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:16,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:16,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:16,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112788623f21d49b436589a9885a0f8e74c8_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666634749/Put/seqid=0 2024-11-27T00:17:16,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742420_1596 (size=12454) 2024-11-27T00:17:16,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:16,077 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112788623f21d49b436589a9885a0f8e74c8_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112788623f21d49b436589a9885a0f8e74c8_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/fd4b1793258c45f7a3d759b19b117c15, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/fd4b1793258c45f7a3d759b19b117c15 is 175, key is test_row_0/A:col10/1732666634749/Put/seqid=0 2024-11-27T00:17:16,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742421_1597 (size=31255) 2024-11-27T00:17:16,088 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/fd4b1793258c45f7a3d759b19b117c15 2024-11-27T00:17:16,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/1397be352d0e4346a3f7587d633a2d9e is 50, key is test_row_0/B:col10/1732666634749/Put/seqid=0 2024-11-27T00:17:16,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742422_1598 (size=12301) 2024-11-27T00:17:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T00:17:16,452 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/6693b354fb91413caab8a05ed4425c98 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6693b354fb91413caab8a05ed4425c98 2024-11-27T00:17:16,454 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:16,455 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/ebff986fbe784cb0af82ee2e2a879139 is 175, key is test_row_0/A:col10/1732666633589/Put/seqid=0 2024-11-27T00:17:16,460 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/B of 199009045896d5ba974e92cd43d397ed into 6693b354fb91413caab8a05ed4425c98(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:16,460 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:16,460 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/B, priority=13, startTime=1732666636031; duration=0sec 2024-11-27T00:17:16,460 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:16,460 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:16,460 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:16,467 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:16,467 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/C is initiating minor compaction (all files) 2024-11-27T00:17:16,467 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/C in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:16,467 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d25a64c1f5e148b8afe691aff054928c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ead817fa39b143699133c0066a58c190, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/c1fa4065121c46768cd7601729565acf] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=36.2 K 2024-11-27T00:17:16,467 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting d25a64c1f5e148b8afe691aff054928c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732666631195 2024-11-27T00:17:16,468 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting ead817fa39b143699133c0066a58c190, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732666632397 2024-11-27T00:17:16,468 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting c1fa4065121c46768cd7601729565acf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732666633585 2024-11-27T00:17:16,474 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:16,475 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/96df1387aef84ce5b80cf4317958794e is 50, key is test_row_0/C:col10/1732666633589/Put/seqid=0 2024-11-27T00:17:16,501 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/1397be352d0e4346a3f7587d633a2d9e 2024-11-27T00:17:16,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742423_1599 (size=31869) 2024-11-27T00:17:16,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742424_1600 (size=12915) 2024-11-27T00:17:16,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/9a009edc50234f61b631065ed5126033 is 50, key is test_row_0/C:col10/1732666634749/Put/seqid=0 2024-11-27T00:17:16,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742425_1601 (size=12301) 2024-11-27T00:17:16,909 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/ebff986fbe784cb0af82ee2e2a879139 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ebff986fbe784cb0af82ee2e2a879139 2024-11-27T00:17:16,913 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/A of 199009045896d5ba974e92cd43d397ed into ebff986fbe784cb0af82ee2e2a879139(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:16,913 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:16,913 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/A, priority=13, startTime=1732666636030; duration=0sec 2024-11-27T00:17:16,913 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:16,913 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:16,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:16,929 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/96df1387aef84ce5b80cf4317958794e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/96df1387aef84ce5b80cf4317958794e 2024-11-27T00:17:16,933 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/C of 199009045896d5ba974e92cd43d397ed into 96df1387aef84ce5b80cf4317958794e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:16,933 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:16,933 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/C, priority=13, startTime=1732666636031; duration=0sec 2024-11-27T00:17:16,933 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:16,933 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:16,966 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/9a009edc50234f61b631065ed5126033 2024-11-27T00:17:16,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/fd4b1793258c45f7a3d759b19b117c15 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/fd4b1793258c45f7a3d759b19b117c15 2024-11-27T00:17:16,973 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/fd4b1793258c45f7a3d759b19b117c15, entries=150, sequenceid=287, filesize=30.5 K 2024-11-27T00:17:16,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/1397be352d0e4346a3f7587d633a2d9e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1397be352d0e4346a3f7587d633a2d9e 2024-11-27T00:17:16,977 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1397be352d0e4346a3f7587d633a2d9e, entries=150, sequenceid=287, filesize=12.0 K 2024-11-27T00:17:16,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/9a009edc50234f61b631065ed5126033 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9a009edc50234f61b631065ed5126033 2024-11-27T00:17:16,981 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9a009edc50234f61b631065ed5126033, entries=150, sequenceid=287, filesize=12.0 K 2024-11-27T00:17:16,981 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=80.51 KB/82440 for 199009045896d5ba974e92cd43d397ed in 922ms, sequenceid=287, compaction requested=false 2024-11-27T00:17:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-27T00:17:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-27T00:17:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:16,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:17:16,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:16,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:16,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:16,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:16,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:16,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:16,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-27T00:17:16,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9960 sec 2024-11-27T00:17:16,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.0020 sec 2024-11-27T00:17:16,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cac4b3fbf42a421b8ad25642d92d3424_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666636977/Put/seqid=0 2024-11-27T00:17:16,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742426_1602 (size=12454) 2024-11-27T00:17:17,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666697043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666697047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-27T00:17:17,134 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:17,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-27T00:17:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T00:17:17,136 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:17,136 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:17,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666697154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666697154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T00:17:17,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T00:17:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:17,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:17,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666697365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666697365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,396 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:17,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cac4b3fbf42a421b8ad25642d92d3424_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cac4b3fbf42a421b8ad25642d92d3424_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:17,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/132f53bb56da4c52af2260e8a667a3fd, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:17,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/132f53bb56da4c52af2260e8a667a3fd is 175, key is test_row_0/A:col10/1732666636977/Put/seqid=0 2024-11-27T00:17:17,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742427_1603 (size=31255) 2024-11-27T00:17:17,430 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/132f53bb56da4c52af2260e8a667a3fd 2024-11-27T00:17:17,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/f47aacb9cf2749eba4d0f74dab0865ab is 50, key is test_row_0/B:col10/1732666636977/Put/seqid=0 2024-11-27T00:17:17,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T00:17:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T00:17:17,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742428_1604 (size=12301) 2024-11-27T00:17:17,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/f47aacb9cf2749eba4d0f74dab0865ab 2024-11-27T00:17:17,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/e2cddff7b70c444e84155240162cf279 is 50, key is test_row_0/C:col10/1732666636977/Put/seqid=0 2024-11-27T00:17:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742429_1605 (size=12301) 2024-11-27T00:17:17,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/e2cddff7b70c444e84155240162cf279 2024-11-27T00:17:17,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/132f53bb56da4c52af2260e8a667a3fd as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/132f53bb56da4c52af2260e8a667a3fd 2024-11-27T00:17:17,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/132f53bb56da4c52af2260e8a667a3fd, entries=150, sequenceid=306, filesize=30.5 K 2024-11-27T00:17:17,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/f47aacb9cf2749eba4d0f74dab0865ab as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f47aacb9cf2749eba4d0f74dab0865ab 2024-11-27T00:17:17,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f47aacb9cf2749eba4d0f74dab0865ab, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T00:17:17,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/e2cddff7b70c444e84155240162cf279 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/e2cddff7b70c444e84155240162cf279 2024-11-27T00:17:17,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/e2cddff7b70c444e84155240162cf279, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T00:17:17,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 199009045896d5ba974e92cd43d397ed in 554ms, sequenceid=306, compaction requested=true 2024-11-27T00:17:17,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:17,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:17,538 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:17,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:17,538 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:17,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:17,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:17,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:17:17,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/A is initiating minor compaction (all files) 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/B is initiating minor compaction (all files) 2024-11-27T00:17:17,539 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/A in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,539 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/B in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,539 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ebff986fbe784cb0af82ee2e2a879139, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/fd4b1793258c45f7a3d759b19b117c15, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/132f53bb56da4c52af2260e8a667a3fd] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=92.2 K 2024-11-27T00:17:17,539 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6693b354fb91413caab8a05ed4425c98, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1397be352d0e4346a3f7587d633a2d9e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f47aacb9cf2749eba4d0f74dab0865ab] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=36.6 K 2024-11-27T00:17:17,539 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ebff986fbe784cb0af82ee2e2a879139, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/fd4b1793258c45f7a3d759b19b117c15, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/132f53bb56da4c52af2260e8a667a3fd] 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6693b354fb91413caab8a05ed4425c98, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732666633585 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebff986fbe784cb0af82ee2e2a879139, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732666633585 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1397be352d0e4346a3f7587d633a2d9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732666634749 2024-11-27T00:17:17,539 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd4b1793258c45f7a3d759b19b117c15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732666634749 2024-11-27T00:17:17,540 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f47aacb9cf2749eba4d0f74dab0865ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732666636945 2024-11-27T00:17:17,540 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 132f53bb56da4c52af2260e8a667a3fd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732666636945 2024-11-27T00:17:17,547 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#B#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:17,547 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/9eacefd29454483b918de20f56d53525 is 50, key is test_row_0/B:col10/1732666636977/Put/seqid=0 2024-11-27T00:17:17,551 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:17,557 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127e05fc4161b964186b0a723456784cf0f_199009045896d5ba974e92cd43d397ed store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:17,559 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127e05fc4161b964186b0a723456784cf0f_199009045896d5ba974e92cd43d397ed, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:17,559 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e05fc4161b964186b0a723456784cf0f_199009045896d5ba974e92cd43d397ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:17,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742430_1606 (size=13017) 2024-11-27T00:17:17,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742431_1607 (size=4469) 2024-11-27T00:17:17,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-27T00:17:17,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,595 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-27T00:17:17,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:17,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:17,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276eb65ec3260c4fcc9d445fb484505cf0_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666637032/Put/seqid=0 2024-11-27T00:17:17,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742432_1608 (size=12454) 2024-11-27T00:17:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:17,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666697693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666697694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666697697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T00:17:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666697803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666697804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666697804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:17,971 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/9eacefd29454483b918de20f56d53525 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/9eacefd29454483b918de20f56d53525 2024-11-27T00:17:17,971 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#A#compaction#513 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:17,972 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/f87123caf5034eb28e3392d14b6f8604 is 175, key is test_row_0/A:col10/1732666636977/Put/seqid=0 2024-11-27T00:17:17,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742433_1609 (size=31971) 2024-11-27T00:17:17,975 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/B of 199009045896d5ba974e92cd43d397ed into 9eacefd29454483b918de20f56d53525(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:17,975 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:17,975 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/B, priority=13, startTime=1732666637538; duration=0sec 2024-11-27T00:17:17,975 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:17,975 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:17,975 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:17,976 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:17,976 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/C is initiating minor compaction (all files) 2024-11-27T00:17:17,976 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/C in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:17,976 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/96df1387aef84ce5b80cf4317958794e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9a009edc50234f61b631065ed5126033, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/e2cddff7b70c444e84155240162cf279] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=36.6 K 2024-11-27T00:17:17,976 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 96df1387aef84ce5b80cf4317958794e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732666633585 2024-11-27T00:17:17,977 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a009edc50234f61b631065ed5126033, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732666634749 2024-11-27T00:17:17,977 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e2cddff7b70c444e84155240162cf279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732666636945 2024-11-27T00:17:17,983 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#C#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:17,983 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/66a368b9339b452988ca2353f328150c is 50, key is test_row_0/C:col10/1732666636977/Put/seqid=0 2024-11-27T00:17:17,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742434_1610 (size=13017) 2024-11-27T00:17:18,002 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/66a368b9339b452988ca2353f328150c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/66a368b9339b452988ca2353f328150c 2024-11-27T00:17:18,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:18,014 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/C of 199009045896d5ba974e92cd43d397ed into 66a368b9339b452988ca2353f328150c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:18,014 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:18,014 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/C, priority=13, startTime=1732666637538; duration=0sec 2024-11-27T00:17:18,014 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:18,014 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666698014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666698014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666698014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,017 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276eb65ec3260c4fcc9d445fb484505cf0_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276eb65ec3260c4fcc9d445fb484505cf0_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/e590532b2c524e06ae4ad406fe971778, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/e590532b2c524e06ae4ad406fe971778 is 175, key is test_row_0/A:col10/1732666637032/Put/seqid=0 2024-11-27T00:17:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742435_1611 (size=31255) 2024-11-27T00:17:18,023 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/e590532b2c524e06ae4ad406fe971778 2024-11-27T00:17:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/34c67aaf66504776ac913bcb16306b63 is 50, key is test_row_0/B:col10/1732666637032/Put/seqid=0 2024-11-27T00:17:18,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742436_1612 (size=12301) 2024-11-27T00:17:18,038 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/34c67aaf66504776ac913bcb16306b63 2024-11-27T00:17:18,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/dfa9f64a24b54cc98f1dd95da73f54e0 is 50, key is test_row_0/C:col10/1732666637032/Put/seqid=0 2024-11-27T00:17:18,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742437_1613 (size=12301) 2024-11-27T00:17:18,066 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/dfa9f64a24b54cc98f1dd95da73f54e0 2024-11-27T00:17:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/e590532b2c524e06ae4ad406fe971778 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/e590532b2c524e06ae4ad406fe971778 2024-11-27T00:17:18,078 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/e590532b2c524e06ae4ad406fe971778, entries=150, sequenceid=322, filesize=30.5 K 2024-11-27T00:17:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/34c67aaf66504776ac913bcb16306b63 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/34c67aaf66504776ac913bcb16306b63 2024-11-27T00:17:18,087 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/34c67aaf66504776ac913bcb16306b63, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T00:17:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/dfa9f64a24b54cc98f1dd95da73f54e0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/dfa9f64a24b54cc98f1dd95da73f54e0 2024-11-27T00:17:18,091 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/dfa9f64a24b54cc98f1dd95da73f54e0, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T00:17:18,093 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 199009045896d5ba974e92cd43d397ed in 498ms, sequenceid=322, compaction requested=false 2024-11-27T00:17:18,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:18,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-27T00:17:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-27T00:17:18,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-27T00:17:18,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 958 msec 2024-11-27T00:17:18,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 962 msec 2024-11-27T00:17:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T00:17:18,276 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:18,277 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T00:17:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-27T00:17:18,278 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T00:17:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T00:17:18,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T00:17:18,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T00:17:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:18,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-27T00:17:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:18,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ef8a2c007945444eb59a60317ec34eee_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666637692/Put/seqid=0 2024-11-27T00:17:18,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742438_1614 (size=12454) 2024-11-27T00:17:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666698346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666698347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666698353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,379 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/f87123caf5034eb28e3392d14b6f8604 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/f87123caf5034eb28e3392d14b6f8604 2024-11-27T00:17:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T00:17:18,387 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/A of 199009045896d5ba974e92cd43d397ed into f87123caf5034eb28e3392d14b6f8604(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:18,387 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:18,387 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/A, priority=13, startTime=1732666637538; duration=0sec 2024-11-27T00:17:18,387 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:18,387 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:18,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T00:17:18,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:18,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666698464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666698464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666698464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T00:17:18,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:18,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T00:17:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666698674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666698674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666698674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:18,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T00:17:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,746 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ef8a2c007945444eb59a60317ec34eee_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ef8a2c007945444eb59a60317ec34eee_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:18,746 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/279a0bcb899c4ea3b78b3596bbc6072b, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:18,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/279a0bcb899c4ea3b78b3596bbc6072b is 175, key is test_row_0/A:col10/1732666637692/Put/seqid=0 2024-11-27T00:17:18,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742439_1615 (size=31255) 2024-11-27T00:17:18,750 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/279a0bcb899c4ea3b78b3596bbc6072b 2024-11-27T00:17:18,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/3417aba0f21c431191f230113a42d210 is 50, key is test_row_0/B:col10/1732666637692/Put/seqid=0 2024-11-27T00:17:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742440_1616 (size=12301) 2024-11-27T00:17:18,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T00:17:18,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:18,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:18,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T00:17:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666698985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666698985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666698985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T00:17:19,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:19,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:19,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:19,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:19,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T00:17:19,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/3417aba0f21c431191f230113a42d210 2024-11-27T00:17:19,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4b7ace71fb4a43c9832d8c9b26d31626 is 50, key is test_row_0/C:col10/1732666637692/Put/seqid=0 2024-11-27T00:17:19,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742441_1617 (size=12301) 2024-11-27T00:17:19,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4b7ace71fb4a43c9832d8c9b26d31626 2024-11-27T00:17:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/279a0bcb899c4ea3b78b3596bbc6072b as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/279a0bcb899c4ea3b78b3596bbc6072b 2024-11-27T00:17:19,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/279a0bcb899c4ea3b78b3596bbc6072b, entries=150, sequenceid=342, filesize=30.5 K 2024-11-27T00:17:19,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/3417aba0f21c431191f230113a42d210 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/3417aba0f21c431191f230113a42d210 2024-11-27T00:17:19,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/3417aba0f21c431191f230113a42d210, entries=150, sequenceid=342, filesize=12.0 K 2024-11-27T00:17:19,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/4b7ace71fb4a43c9832d8c9b26d31626 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4b7ace71fb4a43c9832d8c9b26d31626 2024-11-27T00:17:19,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4b7ace71fb4a43c9832d8c9b26d31626, entries=150, sequenceid=342, filesize=12.0 K 2024-11-27T00:17:19,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 199009045896d5ba974e92cd43d397ed in 867ms, sequenceid=342, compaction requested=true 2024-11-27T00:17:19,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:19,194 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:19,195 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T00:17:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:19,196 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:19,196 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:19,196 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/B is initiating minor compaction (all files) 2024-11-27T00:17:19,196 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/A is initiating minor compaction (all files) 2024-11-27T00:17:19,196 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/B in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:19,196 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/A in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:19,196 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/f87123caf5034eb28e3392d14b6f8604, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/e590532b2c524e06ae4ad406fe971778, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/279a0bcb899c4ea3b78b3596bbc6072b] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=92.3 K 2024-11-27T00:17:19,196 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/9eacefd29454483b918de20f56d53525, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/34c67aaf66504776ac913bcb16306b63, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/3417aba0f21c431191f230113a42d210] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=36.7 K 2024-11-27T00:17:19,196 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:19,196 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. files: [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/f87123caf5034eb28e3392d14b6f8604, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/e590532b2c524e06ae4ad406fe971778, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/279a0bcb899c4ea3b78b3596bbc6072b] 2024-11-27T00:17:19,196 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eacefd29454483b918de20f56d53525, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732666636945 2024-11-27T00:17:19,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting f87123caf5034eb28e3392d14b6f8604, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732666636945 2024-11-27T00:17:19,197 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c67aaf66504776ac913bcb16306b63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732666637032 2024-11-27T00:17:19,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting e590532b2c524e06ae4ad406fe971778, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732666637032 2024-11-27T00:17:19,197 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3417aba0f21c431191f230113a42d210, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732666637692 2024-11-27T00:17:19,197 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] compactions.Compactor(225): Compacting 279a0bcb899c4ea3b78b3596bbc6072b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732666637692 2024-11-27T00:17:19,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-27T00:17:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:19,202 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-27T00:17:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:19,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:19,207 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#B#compaction#521 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:19,207 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/bcc9e60bfdb6430bafd8654abfb7b01a is 50, key is test_row_0/B:col10/1732666637692/Put/seqid=0 2024-11-27T00:17:19,208 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:19,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. as already flushing 2024-11-27T00:17:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:19,213 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b341960a3a6043bbb14c54d6c5396be4_199009045896d5ba974e92cd43d397ed store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:19,215 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b341960a3a6043bbb14c54d6c5396be4_199009045896d5ba974e92cd43d397ed, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:19,216 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b341960a3a6043bbb14c54d6c5396be4_199009045896d5ba974e92cd43d397ed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:19,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742442_1618 (size=13119) 2024-11-27T00:17:19,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fcc740e68ed74e09a3a4c795455e3953_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666638345/Put/seqid=0 2024-11-27T00:17:19,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742443_1619 (size=4469) 2024-11-27T00:17:19,225 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#A#compaction#522 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:19,226 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/7bcc586450594e948707dc9c5ff4ee06 is 175, key is test_row_0/A:col10/1732666637692/Put/seqid=0 2024-11-27T00:17:19,227 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/bcc9e60bfdb6430bafd8654abfb7b01a as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/bcc9e60bfdb6430bafd8654abfb7b01a 2024-11-27T00:17:19,232 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/B of 199009045896d5ba974e92cd43d397ed into bcc9e60bfdb6430bafd8654abfb7b01a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:19,232 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:19,232 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/B, priority=13, startTime=1732666639195; duration=0sec 2024-11-27T00:17:19,232 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T00:17:19,232 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:19,232 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T00:17:19,233 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T00:17:19,233 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1541): 199009045896d5ba974e92cd43d397ed/C is initiating minor compaction (all files) 2024-11-27T00:17:19,233 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 199009045896d5ba974e92cd43d397ed/C in TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:19,233 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/66a368b9339b452988ca2353f328150c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/dfa9f64a24b54cc98f1dd95da73f54e0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4b7ace71fb4a43c9832d8c9b26d31626] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp, totalSize=36.7 K 2024-11-27T00:17:19,234 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a368b9339b452988ca2353f328150c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732666636945 2024-11-27T00:17:19,234 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfa9f64a24b54cc98f1dd95da73f54e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732666637032 2024-11-27T00:17:19,234 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b7ace71fb4a43c9832d8c9b26d31626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732666637692 2024-11-27T00:17:19,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742444_1620 (size=12454) 2024-11-27T00:17:19,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742445_1621 (size=32073) 2024-11-27T00:17:19,240 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 199009045896d5ba974e92cd43d397ed#C#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T00:17:19,241 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/5d412301274747db9972bd05807a0d69 is 50, key is test_row_0/C:col10/1732666637692/Put/seqid=0 2024-11-27T00:17:19,241 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/7bcc586450594e948707dc9c5ff4ee06 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/7bcc586450594e948707dc9c5ff4ee06 2024-11-27T00:17:19,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742446_1622 (size=13119) 2024-11-27T00:17:19,246 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/A of 199009045896d5ba974e92cd43d397ed into 7bcc586450594e948707dc9c5ff4ee06(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:19,246 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:19,246 INFO [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/A, priority=13, startTime=1732666639194; duration=0sec 2024-11-27T00:17:19,247 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:19,247 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:19,247 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/5d412301274747db9972bd05807a0d69 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5d412301274747db9972bd05807a0d69 2024-11-27T00:17:19,249 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 199009045896d5ba974e92cd43d397ed/C of 199009045896d5ba974e92cd43d397ed into 5d412301274747db9972bd05807a0d69(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T00:17:19,249 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:19,250 INFO [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., storeName=199009045896d5ba974e92cd43d397ed/C, priority=13, startTime=1732666639195; duration=0sec 2024-11-27T00:17:19,250 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:19,250 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666699252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666699258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666699363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666699363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T00:17:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666699493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666699493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666699494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666699573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666699573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:19,639 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fcc740e68ed74e09a3a4c795455e3953_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fcc740e68ed74e09a3a4c795455e3953_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:19,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9e6855c81c2c4a9682594fa27799c968, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9e6855c81c2c4a9682594fa27799c968 is 175, key is test_row_0/A:col10/1732666638345/Put/seqid=0 2024-11-27T00:17:19,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742447_1623 (size=31255) 2024-11-27T00:17:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666699884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666699884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:20,049 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9e6855c81c2c4a9682594fa27799c968 2024-11-27T00:17:20,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/97415bc89df64f329ccebcadac98e107 is 50, key is test_row_0/B:col10/1732666638345/Put/seqid=0 2024-11-27T00:17:20,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742448_1624 (size=12301) 2024-11-27T00:17:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43954 deadline: 1732666700394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:20,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43926 deadline: 1732666700394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T00:17:20,464 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/97415bc89df64f329ccebcadac98e107 2024-11-27T00:17:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/efcfd52972c943e2aabc53d41914a96f is 50, key is test_row_0/C:col10/1732666638345/Put/seqid=0 2024-11-27T00:17:20,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742449_1625 (size=12301) 2024-11-27T00:17:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43968 deadline: 1732666700504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:20,505 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_1' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:20,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43924 deadline: 1732666700504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:20,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T00:17:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43942 deadline: 1732666700504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:20,507 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_2' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:20,508 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f036bc4986ea,33881,1732666453890 for 'test_row_0' in 199009045896d5ba974e92cd43d397ed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T00:17:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 , the old value is region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T00:17:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., hostname=f036bc4986ea,33881,1732666453890, seqNum=5 because the exception is null or not the one we care about 2024-11-27T00:17:20,882 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/efcfd52972c943e2aabc53d41914a96f 2024-11-27T00:17:20,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/9e6855c81c2c4a9682594fa27799c968 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9e6855c81c2c4a9682594fa27799c968 2024-11-27T00:17:20,888 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9e6855c81c2c4a9682594fa27799c968, entries=150, sequenceid=357, filesize=30.5 K 2024-11-27T00:17:20,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/97415bc89df64f329ccebcadac98e107 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/97415bc89df64f329ccebcadac98e107 2024-11-27T00:17:20,892 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/97415bc89df64f329ccebcadac98e107, entries=150, sequenceid=357, filesize=12.0 K 2024-11-27T00:17:20,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/efcfd52972c943e2aabc53d41914a96f as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/efcfd52972c943e2aabc53d41914a96f 2024-11-27T00:17:20,908 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/efcfd52972c943e2aabc53d41914a96f, entries=150, sequenceid=357, filesize=12.0 K 2024-11-27T00:17:20,909 INFO [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 199009045896d5ba974e92cd43d397ed in 1707ms, sequenceid=357, compaction requested=false 2024-11-27T00:17:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f036bc4986ea:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-27T00:17:20,911 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-27T00:17:20,911 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:20,911 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,911 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:20,911 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-27T00:17:20,911 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:20,911 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,911 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-27T00:17:20,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:20,912 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-27T00:17:20,912 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:20,912 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:20,912 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,914 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-27T00:17:20,914 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:20,914 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-27T00:17:20,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6330 sec 2024-11-27T00:17:20,914 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-27T00:17:20,914 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:20,914 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,914 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:20,915 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:20,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:20,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.6380 sec 2024-11-27T00:17:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33881 {}] regionserver.HRegion(8855): Flush requested on 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:21,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-27T00:17:21,405 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-27T00:17:21,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:21,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:21,405 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:21,405 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:21,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:21,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:21,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:21,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:21,406 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:21,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:21,415 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-27T00:17:21,415 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:21,415 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:21,415 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:21,416 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:21,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d6cd9fddc9804c00b5e00db3246dc816_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666639250/Put/seqid=0 2024-11-27T00:17:21,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742450_1626 (size=12454) 2024-11-27T00:17:21,823 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:21,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d6cd9fddc9804c00b5e00db3246dc816_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d6cd9fddc9804c00b5e00db3246dc816_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:21,832 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/b8f491fbd6e2417d8e37788f1b070022, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:21,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/b8f491fbd6e2417d8e37788f1b070022 is 175, key is test_row_0/A:col10/1732666639250/Put/seqid=0 2024-11-27T00:17:21,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742451_1627 (size=31255) 2024-11-27T00:17:22,236 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/b8f491fbd6e2417d8e37788f1b070022 2024-11-27T00:17:22,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/d1c14c437ed7406688f3aa2ba7ff8e11 is 50, key is test_row_0/B:col10/1732666639250/Put/seqid=0 2024-11-27T00:17:22,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742452_1628 (size=12301) 2024-11-27T00:17:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-27T00:17:22,445 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:22,519 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-27T00:17:22,519 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:22,519 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:22,520 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:22,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:22,526 INFO [Thread-2337 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2337. 2024-11-27T00:17:22,526 DEBUG [Thread-2337 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:22,526 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:22,526 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:22,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:22,535 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-27T00:17:22,535 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T00:17:22,535 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:22,535 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:22,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:22,535 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T00:17:22,535 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-27T00:17:22,535 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-27T00:17:22,535 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-27T00:17:22,535 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-27T00:17:22,535 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-27T00:17:22,536 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T00:17:22,536 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2976 2024-11-27T00:17:22,536 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3124 2024-11-27T00:17:22,536 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3047 2024-11-27T00:17:22,536 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2962 2024-11-27T00:17:22,536 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3100 2024-11-27T00:17:22,536 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T00:17:22,536 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T00:17:22,536 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:17:22,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:22,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:22,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:22,537 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T00:17:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T00:17:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T00:17:22,541 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666642541"}]},"ts":"1732666642541"} 2024-11-27T00:17:22,543 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T00:17:22,543 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T00:17:22,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T00:17:22,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, UNASSIGN}] 2024-11-27T00:17:22,547 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, UNASSIGN 2024-11-27T00:17:22,548 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=199009045896d5ba974e92cd43d397ed, regionState=CLOSING, regionLocation=f036bc4986ea,33881,1732666453890 2024-11-27T00:17:22,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, UNASSIGN because future has completed 2024-11-27T00:17:22,549 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T00:17:22,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890}] 2024-11-27T00:17:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T00:17:22,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/d1c14c437ed7406688f3aa2ba7ff8e11 2024-11-27T00:17:22,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/bb6161318a544bc192d72edb04da9a1e is 50, key is test_row_0/C:col10/1732666639250/Put/seqid=0 2024-11-27T00:17:22,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742453_1629 (size=12301) 2024-11-27T00:17:22,702 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:22,702 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T00:17:22,702 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 199009045896d5ba974e92cd43d397ed, disabling compactions & flushes 2024-11-27T00:17:22,703 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T00:17:23,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/bb6161318a544bc192d72edb04da9a1e 2024-11-27T00:17:23,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/b8f491fbd6e2417d8e37788f1b070022 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/b8f491fbd6e2417d8e37788f1b070022 2024-11-27T00:17:23,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/b8f491fbd6e2417d8e37788f1b070022, entries=150, sequenceid=379, filesize=30.5 K 2024-11-27T00:17:23,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/d1c14c437ed7406688f3aa2ba7ff8e11 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/d1c14c437ed7406688f3aa2ba7ff8e11 2024-11-27T00:17:23,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/d1c14c437ed7406688f3aa2ba7ff8e11, entries=150, sequenceid=379, filesize=12.0 K 2024-11-27T00:17:23,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/bb6161318a544bc192d72edb04da9a1e as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/bb6161318a544bc192d72edb04da9a1e 2024-11-27T00:17:23,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/bb6161318a544bc192d72edb04da9a1e, entries=150, sequenceid=379, filesize=12.0 K 2024-11-27T00:17:23,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for 199009045896d5ba974e92cd43d397ed in 1674ms, sequenceid=379, compaction requested=true 2024-11-27T00:17:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 199009045896d5ba974e92cd43d397ed: 2024-11-27T00:17:23,079 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T00:17:23,079 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:23,079 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. because compaction request was cancelled 2024-11-27T00:17:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:23,079 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. after waiting 0 ms 2024-11-27T00:17:23,079 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:A 2024-11-27T00:17:23,079 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:23,079 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. because compaction request was cancelled 2024-11-27T00:17:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 199009045896d5ba974e92cd43d397ed:C, priority=-2147483648, current under compaction store size is 2 2024-11-27T00:17:23,079 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. because compaction request was cancelled 2024-11-27T00:17:23,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T00:17:23,080 DEBUG [RS:0;f036bc4986ea:33881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:C 2024-11-27T00:17:23,080 DEBUG [RS:0;f036bc4986ea:33881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 199009045896d5ba974e92cd43d397ed:B 2024-11-27T00:17:23,080 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 199009045896d5ba974e92cd43d397ed 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-27T00:17:23,080 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=A 2024-11-27T00:17:23,080 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:23,080 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=B 2024-11-27T00:17:23,080 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:23,080 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 199009045896d5ba974e92cd43d397ed, store=C 2024-11-27T00:17:23,080 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-27T00:17:23,083 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b23229bd60754c3a901bb9848b48081b_199009045896d5ba974e92cd43d397ed is 50, key is test_row_0/A:col10/1732666642534/Put/seqid=0 2024-11-27T00:17:23,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742454_1630 (size=12454) 2024-11-27T00:17:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T00:17:23,486 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T00:17:23,489 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b23229bd60754c3a901bb9848b48081b_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b23229bd60754c3a901bb9848b48081b_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:23,490 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/24dae2a2d4304aabb4ffeadbdd6aaa96, store: [table=TestAcidGuarantees family=A region=199009045896d5ba974e92cd43d397ed] 2024-11-27T00:17:23,490 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/24dae2a2d4304aabb4ffeadbdd6aaa96 is 175, key is test_row_0/A:col10/1732666642534/Put/seqid=0 2024-11-27T00:17:23,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742455_1631 (size=31255) 2024-11-27T00:17:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T00:17:23,894 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/24dae2a2d4304aabb4ffeadbdd6aaa96 2024-11-27T00:17:23,903 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/94208917c654454699bbc972a21caee9 is 50, key is test_row_0/B:col10/1732666642534/Put/seqid=0 2024-11-27T00:17:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742456_1632 (size=12301) 2024-11-27T00:17:24,308 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/94208917c654454699bbc972a21caee9 2024-11-27T00:17:24,319 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/2baf68f654d54c2cb62176a51be3f376 is 50, key is test_row_0/C:col10/1732666642534/Put/seqid=0 2024-11-27T00:17:24,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742457_1633 (size=12301) 2024-11-27T00:17:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T00:17:24,724 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/2baf68f654d54c2cb62176a51be3f376 2024-11-27T00:17:24,727 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/A/24dae2a2d4304aabb4ffeadbdd6aaa96 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/24dae2a2d4304aabb4ffeadbdd6aaa96 2024-11-27T00:17:24,730 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/24dae2a2d4304aabb4ffeadbdd6aaa96, entries=150, sequenceid=386, filesize=30.5 K 2024-11-27T00:17:24,731 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/B/94208917c654454699bbc972a21caee9 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/94208917c654454699bbc972a21caee9 2024-11-27T00:17:24,736 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/94208917c654454699bbc972a21caee9, entries=150, sequenceid=386, filesize=12.0 K 2024-11-27T00:17:24,736 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/.tmp/C/2baf68f654d54c2cb62176a51be3f376 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/2baf68f654d54c2cb62176a51be3f376 2024-11-27T00:17:24,740 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/2baf68f654d54c2cb62176a51be3f376, entries=150, sequenceid=386, filesize=12.0 K 2024-11-27T00:17:24,741 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 199009045896d5ba974e92cd43d397ed in 1662ms, sequenceid=386, compaction requested=true 2024-11-27T00:17:24,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ae3bc999367949baa89b99791a7dee17, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e697e2fb8b042db8887bce19768dd81, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/0e8cb4b523c341678059cc4a6f7b0f4a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ddf73f6c26f04bbc8d438ff652c834b2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5fa3b9b9d50949209ecebe0a3181d8d3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a22f19cc391b4922a43908e12a0f5d9f, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a946260054104508af9df04aa766d3d2, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5985519833db4597b3667643e42ad022, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9d770b0e9b924a5bba467bfff94ecfa3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9442071aa0514933838ed65f9f8dc127, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/8eabba9d1580421e9b946c2138197301, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/6721e72659db4585925285054aea881e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/30583206c6fe402ca82b36341a6ab911, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e693b35f51a48a595e9fd402bf122f7, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/66dcf346912943d29779b7ad20ccf409, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/260f61870f3842a99c56dac085ced5a0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/930c854fa97c4602a0eb9799e4625319, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/70869a8b75b844ef9803970cfb5ad49a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ebff986fbe784cb0af82ee2e2a879139, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/fd4b1793258c45f7a3d759b19b117c15, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/f87123caf5034eb28e3392d14b6f8604, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/132f53bb56da4c52af2260e8a667a3fd, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/e590532b2c524e06ae4ad406fe971778, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/279a0bcb899c4ea3b78b3596bbc6072b] to archive 2024-11-27T00:17:24,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:17:24,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ae3bc999367949baa89b99791a7dee17 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ae3bc999367949baa89b99791a7dee17 2024-11-27T00:17:24,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e697e2fb8b042db8887bce19768dd81 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e697e2fb8b042db8887bce19768dd81 2024-11-27T00:17:24,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/0e8cb4b523c341678059cc4a6f7b0f4a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/0e8cb4b523c341678059cc4a6f7b0f4a 2024-11-27T00:17:24,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ddf73f6c26f04bbc8d438ff652c834b2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ddf73f6c26f04bbc8d438ff652c834b2 2024-11-27T00:17:24,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5fa3b9b9d50949209ecebe0a3181d8d3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5fa3b9b9d50949209ecebe0a3181d8d3 2024-11-27T00:17:24,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a22f19cc391b4922a43908e12a0f5d9f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a22f19cc391b4922a43908e12a0f5d9f 2024-11-27T00:17:24,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a946260054104508af9df04aa766d3d2 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/a946260054104508af9df04aa766d3d2 2024-11-27T00:17:24,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5985519833db4597b3667643e42ad022 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/5985519833db4597b3667643e42ad022 2024-11-27T00:17:24,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9d770b0e9b924a5bba467bfff94ecfa3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9d770b0e9b924a5bba467bfff94ecfa3 2024-11-27T00:17:24,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9442071aa0514933838ed65f9f8dc127 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9442071aa0514933838ed65f9f8dc127 2024-11-27T00:17:24,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/8eabba9d1580421e9b946c2138197301 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/8eabba9d1580421e9b946c2138197301 2024-11-27T00:17:24,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/6721e72659db4585925285054aea881e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/6721e72659db4585925285054aea881e 2024-11-27T00:17:24,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/30583206c6fe402ca82b36341a6ab911 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/30583206c6fe402ca82b36341a6ab911 2024-11-27T00:17:24,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e693b35f51a48a595e9fd402bf122f7 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/4e693b35f51a48a595e9fd402bf122f7 2024-11-27T00:17:24,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/66dcf346912943d29779b7ad20ccf409 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/66dcf346912943d29779b7ad20ccf409 2024-11-27T00:17:24,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/260f61870f3842a99c56dac085ced5a0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/260f61870f3842a99c56dac085ced5a0 2024-11-27T00:17:24,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/930c854fa97c4602a0eb9799e4625319 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/930c854fa97c4602a0eb9799e4625319 2024-11-27T00:17:24,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/70869a8b75b844ef9803970cfb5ad49a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/70869a8b75b844ef9803970cfb5ad49a 2024-11-27T00:17:24,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ebff986fbe784cb0af82ee2e2a879139 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/ebff986fbe784cb0af82ee2e2a879139 2024-11-27T00:17:24,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/fd4b1793258c45f7a3d759b19b117c15 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/fd4b1793258c45f7a3d759b19b117c15 2024-11-27T00:17:24,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/f87123caf5034eb28e3392d14b6f8604 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/f87123caf5034eb28e3392d14b6f8604 2024-11-27T00:17:24,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/132f53bb56da4c52af2260e8a667a3fd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/132f53bb56da4c52af2260e8a667a3fd 2024-11-27T00:17:24,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/e590532b2c524e06ae4ad406fe971778 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/e590532b2c524e06ae4ad406fe971778 2024-11-27T00:17:24,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/279a0bcb899c4ea3b78b3596bbc6072b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/279a0bcb899c4ea3b78b3596bbc6072b 2024-11-27T00:17:24,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6ce336cfd92c446682e09843c9cce7ea, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/0f5d57ebfe8a47fdb37d4e4469f583f8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/05bab48d8868478281792e31a6a41f83, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1e39e95f89a94a9ab085e92fa658c01e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5403256cdc3c490b83004339b93d2e94, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f1f784667f9f4f7ca1ca18ff7cefece1, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/63492c53c92a4e9ba5afe00f846662af, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/8feebca6c61e46a6bdd3e4c76aa56bcc, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/14d77c5513af43a4965ec17029cdc427, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/344620b4b0ea44fbba9b1bb65ede621d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6828083ccd7e4a4da7db89407d4bfb90, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/91be6451d8d04cf680d9b7566f4c275d, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/147b1908fce441639383d77193f863dd, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5cb344dfc5124074bb7ca978f2287eab, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/be6cc36ac0c74d87a4d8234d5971ad70, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/e1bd0a100691484e82b63fd071c60c55, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/45202a6f3e0c4b16982a25c062ceb9e3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6693b354fb91413caab8a05ed4425c98, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/73ea5f276c104463bd8efe770818f7a0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1397be352d0e4346a3f7587d633a2d9e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/9eacefd29454483b918de20f56d53525, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f47aacb9cf2749eba4d0f74dab0865ab, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/34c67aaf66504776ac913bcb16306b63, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/3417aba0f21c431191f230113a42d210] to archive 2024-11-27T00:17:24,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:17:24,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6ce336cfd92c446682e09843c9cce7ea to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6ce336cfd92c446682e09843c9cce7ea 2024-11-27T00:17:24,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/0f5d57ebfe8a47fdb37d4e4469f583f8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/0f5d57ebfe8a47fdb37d4e4469f583f8 2024-11-27T00:17:24,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/05bab48d8868478281792e31a6a41f83 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/05bab48d8868478281792e31a6a41f83 2024-11-27T00:17:24,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1e39e95f89a94a9ab085e92fa658c01e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1e39e95f89a94a9ab085e92fa658c01e 2024-11-27T00:17:24,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5403256cdc3c490b83004339b93d2e94 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5403256cdc3c490b83004339b93d2e94 2024-11-27T00:17:24,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f1f784667f9f4f7ca1ca18ff7cefece1 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f1f784667f9f4f7ca1ca18ff7cefece1 2024-11-27T00:17:24,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/63492c53c92a4e9ba5afe00f846662af to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/63492c53c92a4e9ba5afe00f846662af 2024-11-27T00:17:24,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/8feebca6c61e46a6bdd3e4c76aa56bcc to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/8feebca6c61e46a6bdd3e4c76aa56bcc 2024-11-27T00:17:24,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/14d77c5513af43a4965ec17029cdc427 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/14d77c5513af43a4965ec17029cdc427 2024-11-27T00:17:24,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/344620b4b0ea44fbba9b1bb65ede621d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/344620b4b0ea44fbba9b1bb65ede621d 2024-11-27T00:17:24,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6828083ccd7e4a4da7db89407d4bfb90 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6828083ccd7e4a4da7db89407d4bfb90 2024-11-27T00:17:24,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/91be6451d8d04cf680d9b7566f4c275d to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/91be6451d8d04cf680d9b7566f4c275d 2024-11-27T00:17:24,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/147b1908fce441639383d77193f863dd to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/147b1908fce441639383d77193f863dd 2024-11-27T00:17:24,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5cb344dfc5124074bb7ca978f2287eab to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/5cb344dfc5124074bb7ca978f2287eab 2024-11-27T00:17:24,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/be6cc36ac0c74d87a4d8234d5971ad70 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/be6cc36ac0c74d87a4d8234d5971ad70 2024-11-27T00:17:24,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/e1bd0a100691484e82b63fd071c60c55 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/e1bd0a100691484e82b63fd071c60c55 2024-11-27T00:17:24,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/45202a6f3e0c4b16982a25c062ceb9e3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/45202a6f3e0c4b16982a25c062ceb9e3 2024-11-27T00:17:24,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6693b354fb91413caab8a05ed4425c98 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/6693b354fb91413caab8a05ed4425c98 2024-11-27T00:17:24,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/73ea5f276c104463bd8efe770818f7a0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/73ea5f276c104463bd8efe770818f7a0 2024-11-27T00:17:24,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1397be352d0e4346a3f7587d633a2d9e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/1397be352d0e4346a3f7587d633a2d9e 2024-11-27T00:17:24,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/9eacefd29454483b918de20f56d53525 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/9eacefd29454483b918de20f56d53525 2024-11-27T00:17:24,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f47aacb9cf2749eba4d0f74dab0865ab to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/f47aacb9cf2749eba4d0f74dab0865ab 2024-11-27T00:17:24,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/34c67aaf66504776ac913bcb16306b63 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/34c67aaf66504776ac913bcb16306b63 2024-11-27T00:17:24,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/3417aba0f21c431191f230113a42d210 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/3417aba0f21c431191f230113a42d210 2024-11-27T00:17:24,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1ef765f3ac7841438fae5fc852b70b26, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/6298175c826c4a82a6327517e0a0426b, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/81ba8c22f61547a39e1e0234bcd5d825, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/72b04f9ed3334ec8b89ce3487f9393e8, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1a4ec1f11c5f48b18ee1169574549629, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/0135205be9804119873717a342f2ec83, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9c58b0b171474e588f7639ec55c197b0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d84c8dd714d945b3b0d51731b3050a62, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/09bae8e7ddc24efbbb6294eae6b7b882, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ffacd8d6fe464b839c1e5237c4f36575, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4d2c6afcc66244c98821ea1126ece7a3, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/434fc1a2cc7f47f691face04bc0e6e8a, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4e873e954026498197ef67a341bd7130, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5aa514c82ea74dcba02dedbee8892e1e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d25a64c1f5e148b8afe691aff054928c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/33e1938613c243b8bdabfb1d02b972a9, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ead817fa39b143699133c0066a58c190, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/96df1387aef84ce5b80cf4317958794e, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/c1fa4065121c46768cd7601729565acf, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9a009edc50234f61b631065ed5126033, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/66a368b9339b452988ca2353f328150c, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/e2cddff7b70c444e84155240162cf279, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/dfa9f64a24b54cc98f1dd95da73f54e0, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4b7ace71fb4a43c9832d8c9b26d31626] to archive 2024-11-27T00:17:24,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T00:17:24,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1ef765f3ac7841438fae5fc852b70b26 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1ef765f3ac7841438fae5fc852b70b26 2024-11-27T00:17:24,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/6298175c826c4a82a6327517e0a0426b to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/6298175c826c4a82a6327517e0a0426b 2024-11-27T00:17:24,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/81ba8c22f61547a39e1e0234bcd5d825 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/81ba8c22f61547a39e1e0234bcd5d825 2024-11-27T00:17:24,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/72b04f9ed3334ec8b89ce3487f9393e8 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/72b04f9ed3334ec8b89ce3487f9393e8 2024-11-27T00:17:24,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1a4ec1f11c5f48b18ee1169574549629 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/1a4ec1f11c5f48b18ee1169574549629 2024-11-27T00:17:24,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/0135205be9804119873717a342f2ec83 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/0135205be9804119873717a342f2ec83 2024-11-27T00:17:24,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9c58b0b171474e588f7639ec55c197b0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9c58b0b171474e588f7639ec55c197b0 2024-11-27T00:17:24,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d84c8dd714d945b3b0d51731b3050a62 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d84c8dd714d945b3b0d51731b3050a62 2024-11-27T00:17:24,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/09bae8e7ddc24efbbb6294eae6b7b882 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/09bae8e7ddc24efbbb6294eae6b7b882 2024-11-27T00:17:24,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ffacd8d6fe464b839c1e5237c4f36575 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ffacd8d6fe464b839c1e5237c4f36575 2024-11-27T00:17:24,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4d2c6afcc66244c98821ea1126ece7a3 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4d2c6afcc66244c98821ea1126ece7a3 2024-11-27T00:17:24,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/434fc1a2cc7f47f691face04bc0e6e8a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/434fc1a2cc7f47f691face04bc0e6e8a 2024-11-27T00:17:24,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4e873e954026498197ef67a341bd7130 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4e873e954026498197ef67a341bd7130 2024-11-27T00:17:24,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5aa514c82ea74dcba02dedbee8892e1e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5aa514c82ea74dcba02dedbee8892e1e 2024-11-27T00:17:24,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d25a64c1f5e148b8afe691aff054928c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/d25a64c1f5e148b8afe691aff054928c 2024-11-27T00:17:24,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/33e1938613c243b8bdabfb1d02b972a9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/33e1938613c243b8bdabfb1d02b972a9 2024-11-27T00:17:24,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ead817fa39b143699133c0066a58c190 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/ead817fa39b143699133c0066a58c190 2024-11-27T00:17:24,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/96df1387aef84ce5b80cf4317958794e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/96df1387aef84ce5b80cf4317958794e 2024-11-27T00:17:24,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/c1fa4065121c46768cd7601729565acf to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/c1fa4065121c46768cd7601729565acf 2024-11-27T00:17:24,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9a009edc50234f61b631065ed5126033 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/9a009edc50234f61b631065ed5126033 2024-11-27T00:17:24,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/66a368b9339b452988ca2353f328150c to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/66a368b9339b452988ca2353f328150c 2024-11-27T00:17:24,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/e2cddff7b70c444e84155240162cf279 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/e2cddff7b70c444e84155240162cf279 2024-11-27T00:17:24,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/dfa9f64a24b54cc98f1dd95da73f54e0 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/dfa9f64a24b54cc98f1dd95da73f54e0 2024-11-27T00:17:24,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4b7ace71fb4a43c9832d8c9b26d31626 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/4b7ace71fb4a43c9832d8c9b26d31626 2024-11-27T00:17:24,811 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/recovered.edits/389.seqid, newMaxSeqId=389, maxSeqId=4 2024-11-27T00:17:24,811 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. 2024-11-27T00:17:24,811 DEBUG [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 199009045896d5ba974e92cd43d397ed: Waiting for close lock at 1732666642702Running coprocessor pre-close hooks at 1732666642702Disabling compacts and flushes for region at 1732666642702Disabling writes for close at 1732666643079 (+377 ms)Obtaining lock to block concurrent updates at 1732666643080 (+1 ms)Preparing flush snapshotting stores in 199009045896d5ba974e92cd43d397ed at 1732666643080Finished memstore snapshotting TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732666643080Flushing stores of TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed. at 1732666643080Flushing 199009045896d5ba974e92cd43d397ed/A: creating writer at 1732666643080Flushing mob file 199009045896d5ba974e92cd43d397ed/A: appending metadata at 1732666643083 (+3 ms)Flushing mob file 199009045896d5ba974e92cd43d397ed/A: closing flushed file at 1732666643083Flushing 199009045896d5ba974e92cd43d397ed/A: appending metadata at 1732666643490 (+407 ms)Flushing 199009045896d5ba974e92cd43d397ed/A: closing flushed file at 1732666643490Flushing 199009045896d5ba974e92cd43d397ed/B: creating writer at 1732666643900 (+410 ms)Flushing 199009045896d5ba974e92cd43d397ed/B: appending metadata at 1732666643902 (+2 ms)Flushing 199009045896d5ba974e92cd43d397ed/B: closing flushed file at 1732666643902Flushing 199009045896d5ba974e92cd43d397ed/C: creating writer at 1732666644316 (+414 ms)Flushing 199009045896d5ba974e92cd43d397ed/C: appending metadata at 1732666644319 (+3 ms)Flushing 199009045896d5ba974e92cd43d397ed/C: closing flushed file at 1732666644319Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67775928: reopening flushed file at 1732666644727 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c140d7b: reopening flushed file at 1732666644730 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dd06cf5: reopening flushed file at 1732666644736 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 199009045896d5ba974e92cd43d397ed in 1662ms, sequenceid=386, compaction requested=true at 1732666644741 (+5 ms)Writing region close event to WAL at 1732666644808 (+67 ms)Running coprocessor post-close hooks at 1732666644811 (+3 ms)Closed at 1732666644811 2024-11-27T00:17:24,814 INFO [RS_CLOSE_REGION-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:24,814 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=199009045896d5ba974e92cd43d397ed, regionState=CLOSED 2024-11-27T00:17:24,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 because future has completed 2024-11-27T00:17:24,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-27T00:17:24,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 199009045896d5ba974e92cd43d397ed, server=f036bc4986ea,33881,1732666453890 in 2.2680 sec 2024-11-27T00:17:24,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-27T00:17:24,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=199009045896d5ba974e92cd43d397ed, UNASSIGN in 2.2730 sec 2024-11-27T00:17:24,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-27T00:17:24,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2780 sec 2024-11-27T00:17:24,825 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732666644825"}]},"ts":"1732666644825"} 2024-11-27T00:17:24,827 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T00:17:24,827 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T00:17:24,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2910 sec 2024-11-27T00:17:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-27T00:17:26,704 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:26,705 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T00:17:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:17:26,707 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:17:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T00:17:26,708 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:17:26,711 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,714 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C, FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/recovered.edits] 2024-11-27T00:17:26,718 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/24dae2a2d4304aabb4ffeadbdd6aaa96 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/24dae2a2d4304aabb4ffeadbdd6aaa96 2024-11-27T00:17:26,719 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/7bcc586450594e948707dc9c5ff4ee06 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/7bcc586450594e948707dc9c5ff4ee06 2024-11-27T00:17:26,721 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9e6855c81c2c4a9682594fa27799c968 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/9e6855c81c2c4a9682594fa27799c968 2024-11-27T00:17:26,723 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/b8f491fbd6e2417d8e37788f1b070022 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/A/b8f491fbd6e2417d8e37788f1b070022 2024-11-27T00:17:26,726 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/94208917c654454699bbc972a21caee9 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/94208917c654454699bbc972a21caee9 2024-11-27T00:17:26,728 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/97415bc89df64f329ccebcadac98e107 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/97415bc89df64f329ccebcadac98e107 2024-11-27T00:17:26,730 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/bcc9e60bfdb6430bafd8654abfb7b01a to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/bcc9e60bfdb6430bafd8654abfb7b01a 2024-11-27T00:17:26,731 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/d1c14c437ed7406688f3aa2ba7ff8e11 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/B/d1c14c437ed7406688f3aa2ba7ff8e11 2024-11-27T00:17:26,735 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/2baf68f654d54c2cb62176a51be3f376 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/2baf68f654d54c2cb62176a51be3f376 2024-11-27T00:17:26,736 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5d412301274747db9972bd05807a0d69 to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/5d412301274747db9972bd05807a0d69 2024-11-27T00:17:26,738 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/bb6161318a544bc192d72edb04da9a1e to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/bb6161318a544bc192d72edb04da9a1e 2024-11-27T00:17:26,740 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/efcfd52972c943e2aabc53d41914a96f to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/C/efcfd52972c943e2aabc53d41914a96f 2024-11-27T00:17:26,744 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/recovered.edits/389.seqid to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed/recovered.edits/389.seqid 2024-11-27T00:17:26,745 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/default/TestAcidGuarantees/199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,745 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T00:17:26,746 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T00:17:26,747 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T00:17:26,752 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270e803ab932ee4bfb80a4bb3bb201b530_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270e803ab932ee4bfb80a4bb3bb201b530_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,754 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272704330a897d4db2910a7bdc481083f3_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272704330a897d4db2910a7bdc481083f3_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,756 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273baa2dcc3c394629b6f751db841d3650_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273baa2dcc3c394629b6f751db841d3650_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,758 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274f487fdfb8314a329ea8fe81953c2009_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411274f487fdfb8314a329ea8fe81953c2009_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,760 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127526cdc93a7474bf793d030fb3d5ded9e_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127526cdc93a7474bf793d030fb3d5ded9e_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,761 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275706c5461da14e5085849102aebaab9b_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275706c5461da14e5085849102aebaab9b_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,762 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276eb65ec3260c4fcc9d445fb484505cf0_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276eb65ec3260c4fcc9d445fb484505cf0_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,764 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276f07d0a2e1464439b47fd53848fc100a_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276f07d0a2e1464439b47fd53848fc100a_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,765 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c273acd12cc453799198d58e4592c6c_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c273acd12cc453799198d58e4592c6c_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,767 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112788623f21d49b436589a9885a0f8e74c8_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112788623f21d49b436589a9885a0f8e74c8_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,768 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278f96da85219d4a12b0555bbc7ed48194_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278f96da85219d4a12b0555bbc7ed48194_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,770 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127917ac97c90304b4f8ea344ffe8618036_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127917ac97c90304b4f8ea344ffe8618036_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,772 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aeac82613160432aba9396fa1fc43350_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127aeac82613160432aba9396fa1fc43350_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,773 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127afd9e82d69eb43c490d855eaf00a168a_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127afd9e82d69eb43c490d855eaf00a168a_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,775 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b23229bd60754c3a901bb9848b48081b_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127b23229bd60754c3a901bb9848b48081b_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,776 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127be96ef9c31734e6d9610888c8a26bbc9_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127be96ef9c31734e6d9610888c8a26bbc9_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,778 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cac4b3fbf42a421b8ad25642d92d3424_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cac4b3fbf42a421b8ad25642d92d3424_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,780 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d6cd9fddc9804c00b5e00db3246dc816_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d6cd9fddc9804c00b5e00db3246dc816_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,781 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ef8a2c007945444eb59a60317ec34eee_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ef8a2c007945444eb59a60317ec34eee_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,783 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fcc740e68ed74e09a3a4c795455e3953_199009045896d5ba974e92cd43d397ed to hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fcc740e68ed74e09a3a4c795455e3953_199009045896d5ba974e92cd43d397ed 2024-11-27T00:17:26,783 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T00:17:26,787 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:17:26,790 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T00:17:26,793 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T00:17:26,794 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:17:26,794 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T00:17:26,795 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732666646794"}]},"ts":"9223372036854775807"} 2024-11-27T00:17:26,797 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T00:17:26,797 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 199009045896d5ba974e92cd43d397ed, NAME => 'TestAcidGuarantees,,1732666618784.199009045896d5ba974e92cd43d397ed.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T00:17:26,797 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T00:17:26,797 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732666646797"}]},"ts":"9223372036854775807"} 2024-11-27T00:17:26,800 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T00:17:26,801 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T00:17:26,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-27T00:17:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-27T00:17:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T00:17:26,815 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T00:17:26,825 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=457 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 401) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2780 (was 2224) - AvailableMemoryMB LEAK? - 2024-11-27T00:17:26,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T00:17:26,825 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T00:17:26,825 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:17:26,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:26,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:26,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T00:17:26,825 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T00:17:26,825 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1201124139, stopped=false 2024-11-27T00:17:26,826 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f036bc4986ea,33599,1732666453115 2024-11-27T00:17:26,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T00:17:26,865 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T00:17:26,865 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:17:26,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:17:26,866 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T00:17:26,867 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T00:17:26,867 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T00:17:26,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T00:17:26,868 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:17:26,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:26,869 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f036bc4986ea,33881,1732666453890' ***** 2024-11-27T00:17:26,869 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T00:17:26,870 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T00:17:26,870 INFO [RS:0;f036bc4986ea:33881 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T00:17:26,870 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T00:17:26,870 INFO [RS:0;f036bc4986ea:33881 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T00:17:26,870 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(959): stopping server f036bc4986ea,33881,1732666453890 2024-11-27T00:17:26,871 INFO [RS:0;f036bc4986ea:33881 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T00:17:26,871 INFO [RS:0;f036bc4986ea:33881 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f036bc4986ea:33881. 2024-11-27T00:17:26,871 DEBUG [RS:0;f036bc4986ea:33881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T00:17:26,871 DEBUG [RS:0;f036bc4986ea:33881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:26,871 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T00:17:26,871 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T00:17:26,871 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T00:17:26,871 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T00:17:26,872 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T00:17:26,872 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T00:17:26,872 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:26,872 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T00:17:26,872 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T00:17:26,872 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T00:17:26,872 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T00:17:26,872 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T00:17:26,873 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-27T00:17:26,873 INFO [regionserver/f036bc4986ea:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T00:17:26,894 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/info/cadcb238708a445a89ccf7ff590b133c is 95, key is TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78./info:/1732666491641/DeleteFamily/seqid=0 2024-11-27T00:17:26,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742458_1634 (size=6514) 2024-11-27T00:17:27,072 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:27,273 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:27,298 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/info/cadcb238708a445a89ccf7ff590b133c 2024-11-27T00:17:27,323 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/ns/f76d473684dd45ebb74e4af745bec3b4 is 93, key is TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78./ns:/1732666491601/DeleteFamily/seqid=0 2024-11-27T00:17:27,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742459_1635 (size=5906) 2024-11-27T00:17:27,473 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:27,673 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:27,727 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/ns/f76d473684dd45ebb74e4af745bec3b4 2024-11-27T00:17:27,748 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/rep_barrier/ea60f059be5b4e73a6231241024420a0 is 102, key is TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78./rep_barrier:/1732666491601/DeleteFamily/seqid=0 2024-11-27T00:17:27,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742460_1636 (size=6025) 2024-11-27T00:17:27,873 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:27,912 INFO [regionserver/f036bc4986ea:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T00:17:27,912 INFO [regionserver/f036bc4986ea:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T00:17:28,073 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:28,152 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/rep_barrier/ea60f059be5b4e73a6231241024420a0 2024-11-27T00:17:28,177 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/table/50b07f61920b42e4a48d7058355e8452 is 96, key is TestAcidGuarantees,,1732666458582.e7356b094232ddb421ff69892486ac78./table:/1732666491601/DeleteFamily/seqid=0 2024-11-27T00:17:28,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742461_1637 (size=5936) 2024-11-27T00:17:28,274 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:28,474 DEBUG [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T00:17:28,580 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/table/50b07f61920b42e4a48d7058355e8452 2024-11-27T00:17:28,583 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/info/cadcb238708a445a89ccf7ff590b133c as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/info/cadcb238708a445a89ccf7ff590b133c 2024-11-27T00:17:28,586 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/info/cadcb238708a445a89ccf7ff590b133c, entries=12, sequenceid=90, filesize=6.4 K 2024-11-27T00:17:28,587 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/ns/f76d473684dd45ebb74e4af745bec3b4 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/ns/f76d473684dd45ebb74e4af745bec3b4 2024-11-27T00:17:28,589 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/ns/f76d473684dd45ebb74e4af745bec3b4, entries=8, sequenceid=90, filesize=5.8 K 2024-11-27T00:17:28,590 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/rep_barrier/ea60f059be5b4e73a6231241024420a0 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/rep_barrier/ea60f059be5b4e73a6231241024420a0 2024-11-27T00:17:28,593 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/rep_barrier/ea60f059be5b4e73a6231241024420a0, entries=6, sequenceid=90, filesize=5.9 K 2024-11-27T00:17:28,594 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/.tmp/table/50b07f61920b42e4a48d7058355e8452 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/table/50b07f61920b42e4a48d7058355e8452 2024-11-27T00:17:28,596 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/table/50b07f61920b42e4a48d7058355e8452, entries=7, sequenceid=90, filesize=5.8 K 2024-11-27T00:17:28,597 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1725ms, sequenceid=90, compaction requested=false 2024-11-27T00:17:28,600 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-27T00:17:28,601 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T00:17:28,601 INFO [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T00:17:28,601 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732666646872Running coprocessor pre-close hooks at 1732666646872Disabling compacts and flushes for region at 1732666646872Disabling writes for close at 1732666646872Obtaining lock to block concurrent updates at 1732666646873 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732666646873Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732666646873Flushing stores of hbase:meta,,1.1588230740 at 1732666646874 (+1 ms)Flushing 1588230740/info: creating writer at 1732666646874Flushing 1588230740/info: appending metadata at 1732666646893 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732666646893Flushing 1588230740/ns: creating writer at 1732666647304 (+411 ms)Flushing 1588230740/ns: appending metadata at 1732666647323 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732666647323Flushing 1588230740/rep_barrier: creating writer at 1732666647731 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732666647748 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732666647748Flushing 1588230740/table: creating writer at 1732666648160 (+412 ms)Flushing 1588230740/table: appending metadata at 1732666648176 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732666648176Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f71b82e: reopening flushed file at 1732666648583 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31872a69: reopening flushed file at 1732666648586 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78bc11aa: reopening flushed file at 1732666648589 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28c45122: reopening flushed file at 1732666648593 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1725ms, sequenceid=90, compaction requested=false at 1732666648597 (+4 ms)Writing region close event to WAL at 1732666648598 (+1 ms)Running coprocessor post-close hooks at 1732666648601 (+3 ms)Closed at 1732666648601 2024-11-27T00:17:28,601 DEBUG [RS_CLOSE_META-regionserver/f036bc4986ea:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T00:17:28,674 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(976): stopping server f036bc4986ea,33881,1732666453890; all regions closed. 2024-11-27T00:17:28,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741834_1010 (size=25344) 2024-11-27T00:17:28,680 DEBUG [RS:0;f036bc4986ea:33881 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/oldWALs 2024-11-27T00:17:28,680 INFO [RS:0;f036bc4986ea:33881 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f036bc4986ea%2C33881%2C1732666453890.meta:.meta(num 1732666458079) 2024-11-27T00:17:28,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741832_1008 (size=14270151) 2024-11-27T00:17:28,686 DEBUG [RS:0;f036bc4986ea:33881 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/oldWALs 2024-11-27T00:17:28,686 INFO [RS:0;f036bc4986ea:33881 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f036bc4986ea%2C33881%2C1732666453890:(num 1732666457016) 2024-11-27T00:17:28,686 DEBUG [RS:0;f036bc4986ea:33881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T00:17:28,686 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T00:17:28,686 INFO [RS:0;f036bc4986ea:33881 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T00:17:28,686 INFO [RS:0;f036bc4986ea:33881 {}] hbase.ChoreService(370): Chore service for: regionserver/f036bc4986ea:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T00:17:28,687 INFO [RS:0;f036bc4986ea:33881 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T00:17:28,687 INFO [regionserver/f036bc4986ea:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T00:17:28,688 INFO [RS:0;f036bc4986ea:33881 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33881 2024-11-27T00:17:28,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T00:17:28,739 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f036bc4986ea,33881,1732666453890 2024-11-27T00:17:28,739 INFO [RS:0;f036bc4986ea:33881 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T00:17:28,741 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f036bc4986ea,33881,1732666453890] 2024-11-27T00:17:28,759 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f036bc4986ea,33881,1732666453890 already deleted, retry=false 2024-11-27T00:17:28,759 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f036bc4986ea,33881,1732666453890 expired; onlineServers=0 2024-11-27T00:17:28,759 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f036bc4986ea,33599,1732666453115' ***** 2024-11-27T00:17:28,759 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T00:17:28,760 INFO [M:0;f036bc4986ea:33599 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T00:17:28,760 INFO [M:0;f036bc4986ea:33599 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T00:17:28,760 DEBUG [M:0;f036bc4986ea:33599 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T00:17:28,760 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T00:17:28,760 DEBUG [M:0;f036bc4986ea:33599 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T00:17:28,760 DEBUG [master/f036bc4986ea:0:becomeActiveMaster-HFileCleaner.large.0-1732666456735 {}] cleaner.HFileCleaner(306): Exit Thread[master/f036bc4986ea:0:becomeActiveMaster-HFileCleaner.large.0-1732666456735,5,FailOnTimeoutGroup] 2024-11-27T00:17:28,760 INFO [M:0;f036bc4986ea:33599 {}] hbase.ChoreService(370): Chore service for: master/f036bc4986ea:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T00:17:28,760 INFO [M:0;f036bc4986ea:33599 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T00:17:28,760 DEBUG [M:0;f036bc4986ea:33599 {}] master.HMaster(1795): Stopping service threads 2024-11-27T00:17:28,761 INFO [M:0;f036bc4986ea:33599 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T00:17:28,760 DEBUG [master/f036bc4986ea:0:becomeActiveMaster-HFileCleaner.small.0-1732666456737 {}] cleaner.HFileCleaner(306): Exit Thread[master/f036bc4986ea:0:becomeActiveMaster-HFileCleaner.small.0-1732666456737,5,FailOnTimeoutGroup] 2024-11-27T00:17:28,761 INFO [M:0;f036bc4986ea:33599 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T00:17:28,761 ERROR [M:0;f036bc4986ea:33599 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:40817 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40817,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-27T00:17:28,762 INFO [M:0;f036bc4986ea:33599 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T00:17:28,762 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T00:17:28,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T00:17:28,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T00:17:28,770 DEBUG [M:0;f036bc4986ea:33599 {}] zookeeper.ZKUtil(347): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T00:17:28,770 WARN [M:0;f036bc4986ea:33599 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T00:17:28,771 INFO [M:0;f036bc4986ea:33599 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/.lastflushedseqids 2024-11-27T00:17:28,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742462_1638 (size=72) 2024-11-27T00:17:28,849 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T00:17:28,849 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33881-0x10179a3e48a0001, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T00:17:28,850 INFO [RS:0;f036bc4986ea:33881 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T00:17:28,850 INFO [RS:0;f036bc4986ea:33881 {}] regionserver.HRegionServer(1031): Exiting; stopping=f036bc4986ea,33881,1732666453890; zookeeper connection closed. 2024-11-27T00:17:28,851 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21799915 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21799915 2024-11-27T00:17:28,852 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T00:17:29,182 INFO [M:0;f036bc4986ea:33599 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T00:17:29,182 INFO [M:0;f036bc4986ea:33599 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T00:17:29,183 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T00:17:29,183 INFO [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T00:17:29,183 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T00:17:29,183 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T00:17:29,183 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T00:17:29,183 INFO [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=769.15 KB heapSize=941.77 KB 2024-11-27T00:17:29,198 DEBUG [M:0;f036bc4986ea:33599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/406be1f4399c4d6aa432dfd9cbad2c6d is 82, key is hbase:meta,,1/info:regioninfo/1732666458159/Put/seqid=0 2024-11-27T00:17:29,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742463_1639 (size=5672) 2024-11-27T00:17:29,601 INFO [M:0;f036bc4986ea:33599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2122 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/406be1f4399c4d6aa432dfd9cbad2c6d 2024-11-27T00:17:29,625 DEBUG [M:0;f036bc4986ea:33599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa33dfaefd9946838462fe87e177c5fc is 2479, key is \x00\x00\x00\x00\x00\x00\x00`/proc:d/1732666557807/Put/seqid=0 2024-11-27T00:17:29,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742464_1640 (size=46388) 2024-11-27T00:17:30,030 INFO [M:0;f036bc4986ea:33599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=768.60 KB at sequenceid=2122 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa33dfaefd9946838462fe87e177c5fc 2024-11-27T00:17:30,034 INFO [M:0;f036bc4986ea:33599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa33dfaefd9946838462fe87e177c5fc 2024-11-27T00:17:30,047 DEBUG [M:0;f036bc4986ea:33599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/481f7105d3634ff9b20a6b84ed6bf921 is 69, key is f036bc4986ea,33881,1732666453890/rs:state/1732666456774/Put/seqid=0 2024-11-27T00:17:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073742465_1641 (size=5156) 2024-11-27T00:17:30,327 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T00:17:30,451 INFO [M:0;f036bc4986ea:33599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2122 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/481f7105d3634ff9b20a6b84ed6bf921 2024-11-27T00:17:30,454 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/406be1f4399c4d6aa432dfd9cbad2c6d as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/406be1f4399c4d6aa432dfd9cbad2c6d 2024-11-27T00:17:30,457 INFO [M:0;f036bc4986ea:33599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/406be1f4399c4d6aa432dfd9cbad2c6d, entries=8, sequenceid=2122, filesize=5.5 K 2024-11-27T00:17:30,458 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa33dfaefd9946838462fe87e177c5fc as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fa33dfaefd9946838462fe87e177c5fc 2024-11-27T00:17:30,460 INFO [M:0;f036bc4986ea:33599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa33dfaefd9946838462fe87e177c5fc 2024-11-27T00:17:30,460 INFO [M:0;f036bc4986ea:33599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fa33dfaefd9946838462fe87e177c5fc, entries=176, sequenceid=2122, filesize=45.3 K 2024-11-27T00:17:30,461 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/481f7105d3634ff9b20a6b84ed6bf921 as hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/481f7105d3634ff9b20a6b84ed6bf921 2024-11-27T00:17:30,463 INFO [M:0;f036bc4986ea:33599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40817/user/jenkins/test-data/b9b7ac72-9219-10cc-2aac-1b2dae3d4772/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/481f7105d3634ff9b20a6b84ed6bf921, entries=1, sequenceid=2122, filesize=5.0 K 2024-11-27T00:17:30,464 INFO [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(3140): Finished flush of dataSize ~769.15 KB/787614, heapSize ~941.47 KB/964064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1281ms, sequenceid=2122, compaction requested=false 2024-11-27T00:17:30,465 INFO [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T00:17:30,465 DEBUG [M:0;f036bc4986ea:33599 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732666649183Disabling compacts and flushes for region at 1732666649183Disabling writes for close at 1732666649183Obtaining lock to block concurrent updates at 1732666649183Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732666649183Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=787614, getHeapSize=964304, getOffHeapSize=0, getCellsCount=2323 at 1732666649183Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732666649184 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732666649184Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732666649197 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732666649197Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732666649607 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732666649625 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732666649625Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732666650034 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732666650046 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732666650046Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ccc98cb: reopening flushed file at 1732666650454 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bf801b1: reopening flushed file at 1732666650457 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29f1a76b: reopening flushed file at 1732666650460 (+3 ms)Finished flush of dataSize ~769.15 KB/787614, heapSize ~941.47 KB/964064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1281ms, sequenceid=2122, compaction requested=false at 1732666650464 (+4 ms)Writing region close event to WAL at 1732666650465 (+1 ms)Closed at 1732666650465 2024-11-27T00:17:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37783 is added to blk_1073741830_1006 (size=926767) 2024-11-27T00:17:30,468 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T00:17:30,468 INFO [M:0;f036bc4986ea:33599 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T00:17:30,468 INFO [M:0;f036bc4986ea:33599 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33599 2024-11-27T00:17:30,469 INFO [M:0;f036bc4986ea:33599 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T00:17:30,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T00:17:30,586 INFO [M:0;f036bc4986ea:33599 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T00:17:30,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33599-0x10179a3e48a0000, quorum=127.0.0.1:56431, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T00:17:30,591 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T00:17:30,595 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T00:17:30,595 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T00:17:30,595 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T00:17:30,595 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/hadoop.log.dir/,STOPPED} 2024-11-27T00:17:30,599 WARN [BP-1139848992-172.17.0.3-1732666449046 heartbeating to localhost/127.0.0.1:40817 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T00:17:30,599 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T00:17:30,599 WARN [BP-1139848992-172.17.0.3-1732666449046 heartbeating to localhost/127.0.0.1:40817 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1139848992-172.17.0.3-1732666449046 (Datanode Uuid 955c775f-b623-4cac-9f03-56e55d61fb75) service to localhost/127.0.0.1:40817 2024-11-27T00:17:30,599 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T00:17:30,604 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/data/data1/current/BP-1139848992-172.17.0.3-1732666449046 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T00:17:30,604 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/cluster_fe38ca85-8c63-2d79-f565-d25c32a7f0a3/data/data2/current/BP-1139848992-172.17.0.3-1732666449046 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T00:17:30,605 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T00:17:30,611 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T00:17:30,611 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T00:17:30,611 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T00:17:30,611 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T00:17:30,611 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ad86aff3-28aa-c20f-cc07-4267ada224f1/hadoop.log.dir/,STOPPED} 2024-11-27T00:17:30,625 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T00:17:30,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down